<!DOCTYPE html>
<html lang="zh-CN">

<head>
  <meta charset="UTF-8">
  <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=2">
  <meta name="theme-color" content="#222">
  <meta name="generator" content="Hexo 4.2.1">
  <link rel="apple-touch-icon" sizes="180x180" href="/images/apple-touch-icon-next.png">
  <link rel="icon" type="image/png" sizes="32x32" href="/images/favicon-32x32-next.png">
  <link rel="icon" type="image/png" sizes="16x16" href="/images/favicon-16x16-next.png">
  <link rel="mask-icon" href="/images/safari-pinned-tab.svg" color="#222">
  <link rel="stylesheet" href="/css/main.css">
  <link rel="stylesheet" href="/lib/font-awesome/css/all.min.css">
  <link rel="stylesheet" href="/lib/pace/pace-theme-minimal.min.css">
  <script src="/lib/pace/pace.min.js"></script>
  <script id="hexo-configurations">
    var NexT = window.NexT ||
    {};
    var CONFIG = {
      "hostname": "cuiqingcai.com",
      "root": "/",
      "scheme": "Pisces",
      "version": "7.8.0",
      "exturl": false,
      "sidebar":
      {
        "position": "right",
        "width": 360,
        "display": "post",
        "padding": 18,
        "offset": 12,
        "onmobile": false,
        "widgets": [
          {
            "type": "image",
            "name": "阿布云",
            "enable": false,
            "url": "https://www.abuyun.com/http-proxy/introduce.html",
            "src": "https://qiniu.cuiqingcai.com/88au8.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "天验",
            "enable": true,
            "url": "https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850",
            "src": "https://qiniu.cuiqingcai.com/bco2a.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "华为云",
            "enable": false,
            "url": "https://activity.huaweicloud.com/2020_618_promotion/index.html?bpName=5f9f98a29e2c40b780c1793086f29fe2&bindType=1&salesID=wangyubei",
            "src": "https://qiniu.cuiqingcai.com/y42ik.jpg",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "张小鸡",
            "enable": false,
            "url": "http://www.zxiaoji.com/",
            "src": "https://qiniu.cuiqingcai.com/fm72f.png",
            "width": "100%"
      },
          {
            "type": "image",
            "name": "Luminati",
            "src": "https://qiniu.cuiqingcai.com/ikkq9.jpg",
            "url": "https://luminati-china.io/?affiliate=ref_5fbbaaa9647883f5c6f77095",
            "width": "100%",
            "enable": false
      },
          {
            "type": "image",
            "name": "IPIDEA",
            "url": "http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc",
            "src": "https://qiniu.cuiqingcai.com/0ywun.png",
            "width": "100%",
            "enable": true
      },
          {
            "type": "tags",
            "name": "标签云",
            "enable": true
      },
          {
            "type": "categories",
            "name": "分类",
            "enable": true
      },
          {
            "type": "friends",
            "name": "友情链接",
            "enable": true
      },
          {
            "type": "hot",
            "name": "猜你喜欢",
            "enable": true
      }]
      },
      "copycode":
      {
        "enable": true,
        "show_result": true,
        "style": "mac"
      },
      "back2top":
      {
        "enable": true,
        "sidebar": false,
        "scrollpercent": true
      },
      "bookmark":
      {
        "enable": false,
        "color": "#222",
        "save": "auto"
      },
      "fancybox": false,
      "mediumzoom": false,
      "lazyload": false,
      "pangu": true,
      "comments":
      {
        "style": "tabs",
        "active": "gitalk",
        "storage": true,
        "lazyload": false,
        "nav": null,
        "activeClass": "gitalk"
      },
      "algolia":
      {
        "hits":
        {
          "per_page": 10
        },
        "labels":
        {
          "input_placeholder": "Search for Posts",
          "hits_empty": "We didn't find any results for the search: ${query}",
          "hits_stats": "${hits} results found in ${time} ms"
        }
      },
      "localsearch":
      {
        "enable": true,
        "trigger": "auto",
        "top_n_per_article": 10,
        "unescape": false,
        "preload": false
      },
      "motion":
      {
        "enable": false,
        "async": false,
        "transition":
        {
          "post_block": "bounceDownIn",
          "post_header": "slideDownIn",
          "post_body": "slideDownIn",
          "coll_header": "slideLeftIn",
          "sidebar": "slideUpIn"
        }
      },
      "path": "search.xml"
    };

  </script>
  <meta name="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:type" content="website">
  <meta property="og:title" content="静觅">
  <meta property="og:url" content="https://cuiqingcai.com/page/17/index.html">
  <meta property="og:site_name" content="静觅">
  <meta property="og:description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
  <meta property="og:locale" content="zh_CN">
  <meta property="article:author" content="崔庆才">
  <meta property="article:tag" content="崔庆才">
  <meta property="article:tag" content="静觅">
  <meta property="article:tag" content="PHP">
  <meta property="article:tag" content="Java">
  <meta property="article:tag" content="Python">
  <meta property="article:tag" content="Spider">
  <meta property="article:tag" content="爬虫">
  <meta property="article:tag" content="Web">
  <meta property="article:tag" content="Kubernetes">
  <meta property="article:tag" content="深度学习">
  <meta property="article:tag" content="机器学习">
  <meta property="article:tag" content="数据分析">
  <meta property="article:tag" content="网络">
  <meta property="article:tag" content="IT">
  <meta property="article:tag" content="技术">
  <meta property="article:tag" content="博客">
  <meta name="twitter:card" content="summary">
  <link rel="canonical" href="https://cuiqingcai.com/page/17/">
  <script id="page-configurations">
    // https://hexo.io/docs/variables.html
    CONFIG.page = {
      sidebar: "",
      isHome: true,
      isPost: false,
      lang: 'zh-CN'
    };

  </script>
  <title>静觅丨崔庆才的个人站点</title>
  <meta name="google-site-verification" content="p_bIcnvirkFzG2dYKuNDivKD8-STet5W7D-01woA2fc" />
  <noscript>
    <style>
      .use-motion .brand,
      .use-motion .menu-item,
      .sidebar-inner,
      .use-motion .post-block,
      .use-motion .pagination,
      .use-motion .comments,
      .use-motion .post-header,
      .use-motion .post-body,
      .use-motion .collection-header
      {
        opacity: initial;
      }

      .use-motion .site-title,
      .use-motion .site-subtitle
      {
        opacity: initial;
        top: initial;
      }

      .use-motion .logo-line-before i
      {
        left: initial;
      }

      .use-motion .logo-line-after i
      {
        right: initial;
      }

    </style>
  </noscript>
  <link rel="alternate" href="/atom.xml" title="静觅" type="application/atom+xml">
</head>

<body itemscope itemtype="http://schema.org/WebPage">
  <div class="container">
    <div class="headband"></div>
    <header class="header" itemscope itemtype="http://schema.org/WPHeader">
      <div class="header-inner">
        <div class="site-brand-container">
          <div class="site-nav-toggle">
            <div class="toggle" aria-label="切换导航栏">
              <span class="toggle-line toggle-line-first"></span>
              <span class="toggle-line toggle-line-middle"></span>
              <span class="toggle-line toggle-line-last"></span>
            </div>
          </div>
          <div class="site-meta">
            <a href="/" class="brand" rel="start">
              <span class="logo-line-before"><i></i></span>
              <h1 class="site-title">静觅 <span class="site-subtitle"> 崔庆才的个人站点 </span>
              </h1>
              <span class="logo-line-after"><i></i></span>
            </a>
          </div>
          <div class="site-nav-right">
            <div class="toggle popup-trigger">
              <i class="fa fa-search fa-fw fa-lg"></i>
            </div>
          </div>
        </div>
        <nav class="site-nav">
          <ul id="menu" class="main-menu menu">
            <li class="menu-item menu-item-home">
              <a href="/" rel="section">首页</a>
            </li>
            <li class="menu-item menu-item-archives">
              <a href="/archives/" rel="section">文章列表</a>
            </li>
            <li class="menu-item menu-item-tags">
              <a href="/tags/" rel="section">文章标签</a>
            </li>
            <li class="menu-item menu-item-categories">
              <a href="/categories/" rel="section">文章分类</a>
            </li>
            <li class="menu-item menu-item-about">
              <a href="/about/" rel="section">关于博主</a>
            </li>
            <li class="menu-item menu-item-message">
              <a href="/message/" rel="section">给我留言</a>
            </li>
            <li class="menu-item menu-item-search">
              <a role="button" class="popup-trigger">搜索 </a>
            </li>
          </ul>
        </nav>
        <div class="search-pop-overlay">
          <div class="popup search-popup">
            <div class="search-header">
              <span class="search-icon">
                <i class="fa fa-search"></i>
              </span>
              <div class="search-input-container">
                <input autocomplete="off" autocapitalize="off" placeholder="搜索..." spellcheck="false" type="search" class="search-input">
              </div>
              <span class="popup-btn-close">
                <i class="fa fa-times-circle"></i>
              </span>
            </div>
            <div id="search-result">
              <div id="no-result">
                <i class="fa fa-spinner fa-pulse fa-5x fa-fw"></i>
              </div>
            </div>
          </div>
        </div>
      </div>
    </header>
    <div class="back-to-top">
      <i class="fa fa-arrow-up"></i>
      <span>0%</span>
    </div>
    <div class="reading-progress-bar"></div>
    <main class="main">
      <div class="main-inner">
        <div class="content-wrap">
          <div class="content index posts-expand">
            <div class="carousel">
              <div id="wowslider-container">
                <div class="ws_images">
                  <ul>
                    <li><a target="_blank" href="https://cuiqingcai.com/5052.html"><img title="Python3网络爬虫开发实战教程" src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a></li>
                    <li><a target="_blank" href="https://t.lagou.com/fRCBRsRCSN6FA"><img title="52讲轻松搞定网络爬虫" src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a></li>
                    <li><a target="_blank" href="https://brightdata.grsm.io/cuiqingcai"><img title="亮网络解锁器" src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/4320.html"><img title="Python3网络爬虫开发视频教程" src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a></li>
                    <li><a target="_blank" href="https://cuiqingcai.com/5094.html"><img title="爬虫代理哪家强？十大付费代理详细对比评测出炉！" src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a></li>
                  </ul>
                </div>
                <div class="ws_thumbs">
                  <div>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/ipy96.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/fqq5e.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/6qnb7.png" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/bjrny.jpg" /></a>
                    <a target="_blank" href="#"><img src="https://qiniu.cuiqingcai.com/nifs6.jpg" /></a>
                  </div>
                </div>
                <div class="ws_shadow"></div>
              </div>
            </div>
            <link rel="stylesheet" href="/lib/wowslide/slide.css">
            <script src="/lib/wowslide/jquery.min.js"></script>
            <script src="/lib/wowslide/slider.js"></script>
            <script>
              jQuery("#wowslider-container").wowSlider(
              {
                effect: "cube",
                prev: "",
                next: "",
                duration: 20 * 100,
                delay: 20 * 100,
                width: 716,
                height: 297,
                autoPlay: true,
                playPause: true,
                stopOnHover: false,
                loop: false,
                bullets: 0,
                caption: true,
                captionEffect: "slide",
                controls: true,
                onBeforeStep: 0,
                images: 0
              });

            </script>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5587.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5587.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.3.2-Redis存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>Redis是一个基于内存的高效的键值型非关系型数据库，存取效率极高，而且支持多种存储数据结构，使用也非常简单。本节中，我们就来介绍一下Python的Redis操作，主要介绍RedisPy这个库的用法。</p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>在开始之前，请确保已经安装好了Redis及RedisPy库。如果要做数据导入/导出操作的话，还需要安装RedisDump。如果没有安装，可以参考第1章。</p>
                  <h2 id="2-Redis和StrictRedis"><a href="#2-Redis和StrictRedis" class="headerlink" title="2. Redis和StrictRedis"></a>2. <code>Redis</code>和<code>StrictRedis</code></h2>
                  <p>RedisPy库提供两个类<code>Redis</code>和<code>StrictRedis</code>来实现Redis的命令操作。</p>
                  <p><code>StrictRedis</code>实现了绝大部分官方的命令，参数也一一对应，比如<code>set()</code>方法就对应Redis命令的<code>set</code>方法。而<code>Redis</code>是<code>StrictRedis</code>的子类，它的主要功能是用于向后兼容旧版本库里的几个方法。为了做兼容，它将方法做了改写，比如<code>lrem()</code>方法就将<code>value</code>和<code>num</code>参数的位置互换，这和Redis命令行的命令参数不一致。</p>
                  <p>官方推荐使用<code>StrictRedis</code>，所以本节中我们也用<code>StrictRedis类</code>的相关方法作演示。</p>
                  <h2 id="3-连接Redis"><a href="#3-连接Redis" class="headerlink" title="3. 连接Redis"></a>3. 连接Redis</h2>
                  <p>现在我们已经在本地安装了Redis并运行在6379端口，密码设置为foobared。那么，可以用如下示例连接Redis并测试：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> redis import StrictRedis</span><br><span class="line"></span><br><span class="line">redis = StrictRedis(<span class="attribute">host</span>=<span class="string">'localhost'</span>, <span class="attribute">port</span>=6379, <span class="attribute">db</span>=0, <span class="attribute">password</span>=<span class="string">'foobared'</span>)</span><br><span class="line">redis.<span class="builtin-name">set</span>(<span class="string">'name'</span>, <span class="string">'Bob'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(redis.<span class="builtin-name">get</span>(<span class="string">'name'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们传入了Redis的地址、运行端口、使用的数据库和密码信息。在默认不传的情况下，这4个参数分别为<code>localhost</code>、<code>6379</code>、<code>0</code>和<code>None</code>。首先声明了一个<code>StrictRedis</code>对象，接下来调用<code>set()</code>方法，设置一个键值对，然后将其获取并打印。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">b'Bob'</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这说明我们连接成功，并可以执行<code>set()</code>和<code>get()</code>操作了。</p>
                  <p>当然，我们还可以使用<code>ConnectionPool</code>来连接，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> redis import StrictRedis, ConnectionPool</span><br><span class="line"></span><br><span class="line">pool = ConnectionPool(<span class="attribute">host</span>=<span class="string">'localhost'</span>, <span class="attribute">port</span>=6379, <span class="attribute">db</span>=0, <span class="attribute">password</span>=<span class="string">'foobared'</span>)</span><br><span class="line">redis = StrictRedis(<span class="attribute">connection_pool</span>=pool)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样的连接效果是一样的。观察源码可以发现，<code>StrictRedis</code>内其实就是用<code>host</code>和<code>port</code>等参数又构造了一个<code>ConnectionPool</code>，所以直接将<code>ConnectionPool</code>当作参数传给<code>StrictRedis</code>也一样。</p>
                  <p>另外，<code>ConnectionPool</code>还支持通过URL来构建。URL的格式支持有如下3种：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="symbol">redis:</span><span class="comment">//[:password]@host:port/db</span></span><br><span class="line"><span class="symbol">rediss:</span><span class="comment">//[:password]@host:port/db</span></span><br><span class="line"><span class="symbol">unix:</span><span class="comment">//[:password]@/path/to/socket.sock?db=db</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这3种URL分别表示创建Redis TCP连接、Redis TCP+SSL连接、Redis UNIX socket连接。我们只需要构造上面任意一种URL即可，其中<code>password</code>部分如果有则可以写，没有则可以省略。下面再用URL连接演示一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">url = <span class="string">'redis://:foobared@localhost:6379/0'</span></span><br><span class="line">pool = ConnectionPool.from_url(url)</span><br><span class="line">redis = StrictRedis(<span class="attribute">connection_pool</span>=pool)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用第一种连接字符串进行连接。首先，声明一个Redis连接字符串，然后调用<code>from_url()</code>方法创建<code>ConnectionPool</code>，接着将其传给<code>StrictRedis</code>即可完成连接，所以使用URL的连接方式还是比较方便的。</p>
                  <h2 id="4-键操作"><a href="#4-键操作" class="headerlink" title="4. 键操作"></a>4. 键操作</h2>
                  <p>表5-5总结了键的一些判断和操作方法。</p>
                  <p>表5-5 键的一些判断和操作方法</p>
                  <p>方法</p>
                  <p>作用</p>
                  <p>参数说明</p>
                  <p>示例</p>
                  <p>示例说明</p>
                  <p>示例结果</p>
                  <p><code>exists(name)</code></p>
                  <p>判断一个键是否存在</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.exists(&#39;name&#39;)</code></p>
                  <p>是否存在<code>name</code>这个键</p>
                  <p><code>True</code></p>
                  <p><code>delete(name)</code></p>
                  <p>删除一个键</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.delete(&#39;name&#39;)</code></p>
                  <p>删除<code>name</code>这个键</p>
                  <p>1</p>
                  <p><code>type(name)</code></p>
                  <p>判断键类型</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.type(&#39;name&#39;)</code></p>
                  <p>判断<code>name</code>这个键类型</p>
                  <p><code>b&#39;string&#39;</code></p>
                  <p><code>keys(pattern)</code></p>
                  <p>获取所有符合规则的键</p>
                  <p><code>pattern</code>：匹配规则</p>
                  <p><code>redis.keys(&#39;n*&#39;)</code></p>
                  <p>获取所有以<code>n</code>开头的键</p>
                  <p><code>[b&#39;name&#39;]</code></p>
                  <p><code>randomkey()</code></p>
                  <p>获取随机的一个键</p>
                  <p><code>randomkey()</code></p>
                  <p>获取随机的一个键</p>
                  <p><code>b&#39;name&#39;</code></p>
                  <p><code>rename(src, dst)</code></p>
                  <p>重命名键</p>
                  <p><code>src</code>：原键名；<code>dst</code>：新键名</p>
                  <p><code>redis.rename(&#39;name&#39;, &#39;nickname&#39;)</code></p>
                  <p>将<code>name</code>重命名为<code>nickname</code></p>
                  <p><code>True</code></p>
                  <p><code>dbsize()</code></p>
                  <p>获取当前数据库中键的数目</p>
                  <p><code>dbsize()</code></p>
                  <p>获取当前数据库中键的数目</p>
                  <p>100</p>
                  <p><code>expire(name, time)</code></p>
                  <p>设定键的过期时间，单位为秒</p>
                  <p><code>name</code>：键名；<code>time</code>：秒数</p>
                  <p><code>redis.expire(&#39;name&#39;, 2)</code></p>
                  <p>将<code>name</code>键的过期时间设置为2秒</p>
                  <p><code>True</code></p>
                  <p><code>ttl(name)</code></p>
                  <p>获取键的过期时间，单位为秒，-1表示永久不过期</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.ttl(&#39;name&#39;)</code></p>
                  <p>获取<code>name</code>这个键的过期时间</p>
                  <p>-1</p>
                  <p><code>move(name, db)</code></p>
                  <p>将键移动到其他数据库</p>
                  <p><code>name</code>：键名；<code>db</code>：数据库代号</p>
                  <p><code>move(&#39;name&#39;, 2)</code></p>
                  <p>将<code>name</code>移动到2号数据库</p>
                  <p><code>True</code></p>
                  <p><code>flushdb()</code></p>
                  <p>删除当前选择数据库中的所有键</p>
                  <p><code>flushdb()</code></p>
                  <p>删除当前选择数据库中的所有键</p>
                  <p><code>True</code></p>
                  <p><code>flushall()</code></p>
                  <p>删除所有数据库中的所有键</p>
                  <p><code>flushall()</code></p>
                  <p>删除所有数据库中的所有键</p>
                  <p><code>True</code></p>
                  <h2 id="5-字符串操作"><a href="#5-字符串操作" class="headerlink" title="5. 字符串操作"></a>5. 字符串操作</h2>
                  <p>Redis支持最基本的键值对形式存储，用法总结如表5-6所示。</p>
                  <p>表5-6 键值对形式存储</p>
                  <p>方法</p>
                  <p>作用</p>
                  <p>参数说明</p>
                  <p>示例</p>
                  <p>示例说明</p>
                  <p>示例结果</p>
                  <p><code>set(name, value)</code></p>
                  <p>给数据库中键为<code>name</code>的<code>string</code>赋予值<code>value</code></p>
                  <p><code>name</code>: 键名；<code>value</code>: 值</p>
                  <p><code>redis.set(&#39;name&#39;, &#39;Bob&#39;)</code></p>
                  <p>给<code>name</code>这个键的<code>value</code>赋值为<code>Bob</code></p>
                  <p><code>True</code></p>
                  <p><code>get(name)</code></p>
                  <p>返回数据库中键为<code>name</code>的<code>string</code>的<code>value</code></p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.get(&#39;name&#39;)</code></p>
                  <p>返回<code>name</code>这个键的<code>value</code></p>
                  <p><code>b&#39;Bob&#39;</code></p>
                  <p><code>getset(name, value)</code></p>
                  <p>给数据库中键为<code>name</code>的<code>string</code>赋予值<code>value</code>并返回上次的<code>value</code></p>
                  <p><code>name</code>：键名；<code>value</code>：新值</p>
                  <p><code>redis.getset(&#39;name&#39;, &#39;Mike&#39;)</code></p>
                  <p>赋值<code>name</code>为<code>Mike</code>并得到上次的<code>value</code></p>
                  <p><code>b&#39;Bob&#39;</code></p>
                  <p><code>mget(keys, *args)</code></p>
                  <p>返回多个键对应的<code>value</code></p>
                  <p><code>keys</code>：键的列表</p>
                  <p><code>redis.mget([&#39;name&#39;, &#39;nickname&#39;])</code></p>
                  <p>返回<code>name</code>和<code>nickname</code>的<code>value</code></p>
                  <p><code>[b&#39;Mike&#39;, b&#39;Miker&#39;]</code></p>
                  <p><code>setnx(name, value)</code></p>
                  <p>如果不存在这个键值对，则更新<code>value</code>，否则不变</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.setnx(&#39;newname&#39;, &#39;James&#39;)</code></p>
                  <p>如果<code>newname</code>这个键不存在，则设置值为<code>James</code></p>
                  <p>第一次运行结果是<code>True</code>，第二次运行结果是<code>False</code></p>
                  <p><code>setex(name, time, value)</code></p>
                  <p>设置可以对应的值为<code>string</code>类型的<code>value</code>，并指定此键值对应的有效期</p>
                  <p><code>name</code>: 键名；<code>time</code>: 有效期； <code>value</code>：值</p>
                  <p><code>redis.setex(&#39;name&#39;, 1, &#39;James&#39;)</code></p>
                  <p>将<code>name</code>这个键的值设为<code>James</code>，有效期为1秒</p>
                  <p><code>True</code></p>
                  <p><code>setrange(name, offset, value)</code></p>
                  <p>设置指定键的<code>value</code>值的子字符串</p>
                  <p><code>name</code>：键名；<code>offset</code>：偏移量；<code>value</code>：值</p>
                  <p><code>redis.set(&#39;name&#39;, &#39;Hello&#39;) redis.setrange(&#39;name&#39;, 6, &#39;World&#39;)</code></p>
                  <p>设置<code>name</code>为<code>Hello</code>字符串，并在<code>index</code>为6的位置补<code>World</code></p>
                  <p>11，修改后的字符串长度</p>
                  <p><code>mset(mapping)</code></p>
                  <p>批量赋值</p>
                  <p><code>mapping</code>：字典</p>
                  <p><code>redis.mset({&#39;name1&#39;: &#39;Durant&#39;, &#39;name2&#39;: &#39;James&#39;})</code></p>
                  <p>将<code>name1</code>设为<code>Durant</code>，<code>name2</code>设为<code>James</code></p>
                  <p><code>True</code></p>
                  <p><code>msetnx(mapping)</code></p>
                  <p>键均不存在时才批量赋值</p>
                  <p><code>mapping</code>：字典</p>
                  <p><code>redis.msetnx({&#39;name3&#39;: &#39;Smith&#39;, &#39;name4&#39;: &#39;Curry&#39;})</code></p>
                  <p>在<code>name3</code>和<code>name4</code>均不存在的情况下才设置二者值</p>
                  <p><code>True</code></p>
                  <p><code>incr(name, amount=1)</code></p>
                  <p>键为<code>name</code>的<code>value</code>增值操作，默认为1，键不存在则被创建并设为<code>amount</code></p>
                  <p><code>name</code>：键名；<code>amount</code>：增长的值</p>
                  <p><code>redis.incr(&#39;age&#39;, 1)</code></p>
                  <p><code>age</code>对应的值增1，若不存在，则会创建并设置为1</p>
                  <p>1，即修改后的值</p>
                  <p><code>decr(name, amount=1)</code></p>
                  <p>键为<code>name</code>的<code>value</code>减值操作，默认为1，键不存在则被创建并将<code>value</code>设置为<code>\-amount</code></p>
                  <p><code>name</code>：键名； <code>amount</code>：减少的值</p>
                  <p><code>redis.decr(&#39;age&#39;, 1)</code></p>
                  <p><code>age</code>对应的值减1，若不存在，则会创建并设置为-1</p>
                  <p>-1，即修改后的值</p>
                  <p><code>append(key, value)</code></p>
                  <p>键为<code>name</code>的<code>string</code>的值附加<code>value</code></p>
                  <p><code>key</code>：键名</p>
                  <p><code>redis.append(&#39;nickname&#39;, &#39;OK&#39;)</code></p>
                  <p>向键为<code>nickname</code>的值后追加<code>OK</code></p>
                  <p>13，即修改后的字符串长度</p>
                  <p><code>substr(name, start, end=-1)</code></p>
                  <p>返回键为<code>name</code>的<code>string</code>的子串</p>
                  <p><code>name</code>：键名；<code>start</code>：起始索引；<code>end</code>：终止索引，默认为-1，表示截取到末尾</p>
                  <p><code>redis.substr(&#39;name&#39;, 1, 4)</code></p>
                  <p>返回键为<code>name</code>的值的字符串，截取索引为1~4的字符</p>
                  <p><code>b&#39;ello&#39;</code></p>
                  <p><code>getrange(key, start, end)</code></p>
                  <p>获取键的<code>value</code>值从<code>start</code>到<code>end</code>的子字符串</p>
                  <p><code>key</code>：键名；<code>start</code>：起始索引；<code>end</code>：终止索引</p>
                  <p><code>redis.getrange(&#39;name&#39;, 1, 4)</code></p>
                  <p>返回键为<code>name</code>的值的字符串，截取索引为1~4的字符</p>
                  <p><code>b&#39;ello&#39;</code></p>
                  <h2 id="6-列表操作"><a href="#6-列表操作" class="headerlink" title="6. 列表操作"></a>6. 列表操作</h2>
                  <p>Redis还提供了列表存储，列表内的元素可以重复，而且可以从两端存储，用法如表5-7所示。</p>
                  <p>表5-7 列表操作</p>
                  <p>方法</p>
                  <p>作用</p>
                  <p>参数说明</p>
                  <p>示例</p>
                  <p>示例说明</p>
                  <p>示例结果</p>
                  <p><code>rpush(name, *values)</code></p>
                  <p>在键为<code>name</code>的列表末尾添加值为<code>value</code>的元素，可以传多个</p>
                  <p><code>name</code>：键名；<code>values</code>：值</p>
                  <p><code>redis.rpush(&#39;list&#39;, 1, 2, 3)</code></p>
                  <p>向键为<code>list</code>的列表尾添加1、2、3</p>
                  <p>3，列表大小</p>
                  <p><code>lpush(name, *values)</code></p>
                  <p>在键为<code>name</code>的列表头添加值为<code>value</code>的元素，可以传多个</p>
                  <p><code>name</code>：键名；<code>values</code>：值</p>
                  <p><code>redis.lpush(&#39;list&#39;, 0)</code></p>
                  <p>向键为<code>list</code>的列表头部添加0</p>
                  <p>4，列表大小</p>
                  <p><code>llen(name)</code></p>
                  <p>返回键为<code>name</code>的列表的长度</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.llen(&#39;list&#39;)</code></p>
                  <p>返回键为<code>list</code>的列表的长度</p>
                  <p>4</p>
                  <p><code>lrange(name, start, end)</code></p>
                  <p>返回键为<code>name</code>的列表中<code>start</code>至<code>end</code>之间的元素</p>
                  <p><code>name</code>：键名；<code>start</code>：起始索引；<code>end</code>：终止索引</p>
                  <p><code>redis.lrange(&#39;list&#39;, 1, 3)</code></p>
                  <p>返回起始索引为1终止索引为3的索引范围对应的列表</p>
                  <p><code>[b&#39;3&#39;, b&#39;2&#39;, b&#39;1&#39;]</code></p>
                  <p><code>ltrim(name, start, end)</code></p>
                  <p>截取键为<code>name</code>的列表，保留索引为<code>start</code>到<code>end</code>的内容</p>
                  <p><code>name</code>：键名；<code>start</code>：起始索引；<code>end</code>：终止索引</p>
                  <p><code>ltrim(&#39;list&#39;, 1, 3)</code></p>
                  <p>保留键为<code>list</code>的索引为1到3的元素</p>
                  <p><code>True</code></p>
                  <p><code>lindex(name, index)</code></p>
                  <p>返回键为<code>name</code>的列表中<code>index</code>位置的元素</p>
                  <p><code>name</code>：键名；<code>index</code>：索引</p>
                  <p><code>redis.lindex(&#39;list&#39;, 1)</code></p>
                  <p>返回键为<code>list</code>的列表索引为1的元素</p>
                  <p>b’2’</p>
                  <p><code>lset(name, index, value)</code></p>
                  <p>给键为<code>name</code>的列表中<code>index</code>位置的元素赋值，越界则报错</p>
                  <p><code>name</code>：键名；<code>index</code>：索引位置；<code>value</code>：值</p>
                  <p><code>redis.lset(&#39;list&#39;, 1, 5)</code></p>
                  <p>将键为<code>list</code>的列表中索引为1的位置赋值为5</p>
                  <p><code>True</code></p>
                  <p><code>lrem(name, count, value)</code></p>
                  <p>删除<code>count</code>个键的列表中值为<code>value</code>的元素</p>
                  <p><code>name</code>：键名；<code>count</code>：删除个数；<code>value</code>：值</p>
                  <p><code>redis.lrem(&#39;list&#39;, 2, 3)</code></p>
                  <p>将键为<code>list</code>的列表删除两个3</p>
                  <p>1，即删除的个数</p>
                  <p><code>lpop(name)</code></p>
                  <p>返回并删除键为<code>name</code>的列表中的首元素</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.lpop(&#39;list&#39;)</code></p>
                  <p>返回并删除名为<code>list</code>的列表中的第一个元素</p>
                  <p><code>b&#39;5&#39;</code></p>
                  <p><code>rpop(name)</code></p>
                  <p>返回并删除键为<code>name</code>的列表中的尾元素</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.rpop(&#39;list&#39;)</code></p>
                  <p>返回并删除名为<code>list</code>的列表中的最后一个元素</p>
                  <p><code>b&#39;2&#39;</code></p>
                  <p><code>blpop(keys, timeout=0)</code></p>
                  <p>返回并删除名称在<code>keys</code>中的<code>list</code>中的首个元素，如果列表为空，则会一直阻塞等待</p>
                  <p><code>keys</code>：键列表；<code>timeout</code>： 超时等待时间，0为一直等待</p>
                  <p><code>redis.blpop(&#39;list&#39;)</code></p>
                  <p>返回并删除键为<code>list</code>的列表中的第一个元素</p>
                  <p><code>[b&#39;5&#39;]</code></p>
                  <p><code>brpop(keys, timeout=0)</code></p>
                  <p>返回并删除键为<code>name</code>的列表中的尾元素，如果<code>list</code>为空，则会一直阻塞等待</p>
                  <p><code>keys</code>：键列表；<code>timeout</code>：超时等待时间，0为一直等待</p>
                  <p><code>redis.brpop(&#39;list&#39;)</code></p>
                  <p>返回并删除名为<code>list</code>的列表中的最后一个元素</p>
                  <p><code>[b&#39;2&#39;]</code></p>
                  <p><code>rpoplpush(src, dst)</code></p>
                  <p>返回并删除名称为<code>src</code>的列表的尾元素，并将该元素添加到名称为<code>dst</code>的列表头部</p>
                  <p><code>src</code>：源列表的键；<code>dst</code>：目标列表的key</p>
                  <p><code>redis.rpoplpush(&#39;list&#39;, &#39;list2&#39;)</code></p>
                  <p>将键为<code>list</code>的列表尾元素删除并将其添加到键为<code>list2</code>的列表头部，然后返回</p>
                  <p><code>b&#39;2&#39;</code></p>
                  <h2 id="7-集合操作"><a href="#7-集合操作" class="headerlink" title="7. 集合操作"></a>7. 集合操作</h2>
                  <p>Redis还提供了集合存储，集合中的元素都是不重复的，用法如表5-8所示。</p>
                  <p>表5-8 集合操作</p>
                  <p>方法</p>
                  <p>作用</p>
                  <p>参数说明</p>
                  <p>示例</p>
                  <p>示例说明</p>
                  <p>示例结果</p>
                  <p><code>sadd(name, *values)</code></p>
                  <p>向键为<code>name</code>的集合中添加元素</p>
                  <p><code>name</code>：键名；<code>values</code>：值，可为多个</p>
                  <p><code>redis.sadd(&#39;tags&#39;, &#39;Book&#39;, &#39;Tea&#39;, &#39;Coffee&#39;)</code></p>
                  <p>向键为<code>tags</code>的集合中添加<code>Book</code>、<code>Tea</code>和<code>Coffee</code>这3个内容</p>
                  <p>3，即插入的数据个数</p>
                  <p><code>srem(name, *values)</code></p>
                  <p>从键为<code>name</code>的集合中删除元素</p>
                  <p><code>name</code>：键名；<code>values</code>：值，可为多个</p>
                  <p><code>redis.srem(&#39;tags&#39;, &#39;Book&#39;)</code></p>
                  <p>从键为<code>tags</code>的集合中删除<code>Book</code></p>
                  <p>1，即删除的数据个数</p>
                  <p><code>spop(name)</code></p>
                  <p>随机返回并删除键为<code>name</code>的集合中的一个元素</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.spop(&#39;tags&#39;)</code></p>
                  <p>从键为<code>tags</code>的集合中随机删除并返回该元素</p>
                  <p><code>b&#39;Tea&#39;</code></p>
                  <p><code>smove(src, dst, value)</code></p>
                  <p>从<code>src</code>对应的集合中移除元素并将其添加到<code>dst</code>对应的集合中</p>
                  <p><code>src</code>：源集合；<code>dst</code>：目标集合；<code>value</code>：元素值</p>
                  <p><code>redis.smove(&#39;tags&#39;, &#39;tags2&#39;, &#39;Coffee&#39;)</code></p>
                  <p>从键为<code>tags</code>的集合中删除元素<code>Coffee</code>并将其添加到键为<code>tags2</code>的集合</p>
                  <p><code>True</code></p>
                  <p><code>scard(name)</code></p>
                  <p>返回键为<code>name</code>的集合的元素个数</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.scard(&#39;tags&#39;)</code></p>
                  <p>获取键为<code>tags</code>的集合中的元素个数</p>
                  <p>3</p>
                  <p><code>sismember(name, value)</code></p>
                  <p>测试<code>member</code>是否是键为<code>name</code>的集合的元素</p>
                  <p><code>name</code>：键值</p>
                  <p><code>redis.sismember(&#39;tags&#39;, &#39;Book&#39;)</code></p>
                  <p>判断<code>Book</code>是否是键为<code>tags</code>的集合元素</p>
                  <p><code>True</code></p>
                  <p><code>sinter(keys, *args)</code></p>
                  <p>返回所有给定键的集合的交集</p>
                  <p><code>keys</code>：键列表</p>
                  <p><code>redis.sinter([&#39;tags&#39;, &#39;tags2&#39;])</code></p>
                  <p>返回键为<code>tags</code>的集合和键为<code>tags2</code>的集合的交集</p>
                  <p><code>{b&#39;Coffee&#39;}</code></p>
                  <p><code>sinterstore(dest, keys, *args)</code></p>
                  <p>求交集并将交集保存到<code>dest</code>的集合</p>
                  <p><code>dest</code>：结果集合；<code>keys</code>：键列表</p>
                  <p><code>redis.sinterstore(&#39;inttag&#39;, [&#39;tags&#39;, &#39;tags2&#39;])</code></p>
                  <p>求键为<code>tags</code>的集合和键为<code>tags2</code>的集合的交集并将其保存为<code>inttag</code></p>
                  <p>1</p>
                  <p><code>sunion(keys, *args)</code></p>
                  <p>返回所有给定键的集合的并集</p>
                  <p><code>keys</code>：键列表</p>
                  <p><code>redis.sunion([&#39;tags&#39;, &#39;tags2&#39;])</code></p>
                  <p>返回键为<code>tags</code>的集合和键为<code>tags2</code>的集合的并集</p>
                  <p><code>{b&#39;Coffee&#39;, b&#39;Book&#39;, b&#39;Pen&#39;}</code></p>
                  <p><code>sunionstore(dest, keys, *args)</code></p>
                  <p>求并集并将并集保存到<code>dest</code>的集合</p>
                  <p><code>dest</code>：结果集合；<code>keys</code>：键列表</p>
                  <p><code>redis.sunionstore(&#39;inttag&#39;, [&#39;tags&#39;, &#39;tags2&#39;])</code></p>
                  <p>求键为<code>tags</code>的集合和键为<code>tags2</code>的集合的并集并将其保存为<code>inttag</code></p>
                  <p>3</p>
                  <p><code>sdiff(keys, *args)</code></p>
                  <p>返回所有给定键的集合的差集</p>
                  <p><code>keys</code>：键列表</p>
                  <p><code>redis.sdiff([&#39;tags&#39;, &#39;tags2&#39;])</code></p>
                  <p>返回键为<code>tags</code>的集合和键为<code>tags2</code>的集合的差集</p>
                  <p><code>{b&#39;Book&#39;, b&#39;Pen&#39;}</code></p>
                  <p><code>sdiffstore(dest, keys, *args)</code></p>
                  <p>求差集并将差集保存到<code>dest</code>集合</p>
                  <p><code>dest</code>：结果集合；<code>keys</code>：键列表</p>
                  <p><code>redis.sdiffstore(&#39;inttag&#39;, [&#39;tags&#39;, &#39;tags2&#39;])</code></p>
                  <p>求键为tags<code>的集合和键为</code>tags2<code>的集合的差集并将其保存为</code>inttag`</p>
                  <p>3</p>
                  <p><code>smembers(name)</code></p>
                  <p>返回键为<code>name</code>的集合的所有元素</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.smembers(&#39;tags&#39;)</code></p>
                  <p>返回键为<code>tags</code>的集合的所有元素</p>
                  <p><code>{b&#39;Pen&#39;, b&#39;Book&#39;, b&#39;Coffee&#39;}</code></p>
                  <p><code>srandmember(name)</code></p>
                  <p>随机返回键为<code>name</code>的集合中的一个元素，但不删除元素</p>
                  <p><code>name</code>：键值</p>
                  <p><code>redis.srandmember(&#39;tags&#39;)</code></p>
                  <p>随机返回键为<code>tags</code>的集合中的一个元素</p>
                  <h2 id="8-有序集合操作"><a href="#8-有序集合操作" class="headerlink" title="8. 有序集合操作"></a>8. 有序集合操作</h2>
                  <p>有序集合比集合多了一个分数字段，利用它可以对集合中的数据进行排序，其用法总结如表5-9所示。</p>
                  <p>表5-9 有序集合操作</p>
                  <p>方法</p>
                  <p>作用</p>
                  <p>参数说明</p>
                  <p>示例</p>
                  <p>示例说明</p>
                  <p>示例结果</p>
                  <p><code>zadd(name, *args, **kwargs)</code></p>
                  <p>向键为<code>name</code>的zset中添加元素member，score用于排序。如果该元素存在，则更新其顺序</p>
                  <p><code>name</code>： 键名；<code>args</code>：可变参数</p>
                  <p><code>redis.zadd(&#39;grade&#39;, 100, &#39;Bob&#39;, 98, &#39;Mike&#39;)</code></p>
                  <p>向键为<code>grade</code>的zset中添加<code>Bob</code>（其<code>score</code>为100），并添加<code>Mike</code>（其<code>score</code>为98）</p>
                  <p>2，即添加的元素个数</p>
                  <p><code>zrem(name, *values)</code></p>
                  <p>删除键为<code>name</code>的zset中的元素</p>
                  <p><code>name</code>：键名；<code>values</code>：元素</p>
                  <p><code>redis.zrem(&#39;grade&#39;, &#39;Mike&#39;)</code></p>
                  <p>从键为<code>grade</code>的zset中删除<code>Mike</code></p>
                  <p>1，即删除的元素个数</p>
                  <p><code>zincrby(name, value, amount=1)</code></p>
                  <p>如果在键为<code>name</code>的zset中已经存在元素<code>value</code>，则将该元素的<code>score</code>增加<code>amount</code>；否则向该集合中添加该元素，其<code>score</code>的值为<code>amount</code></p>
                  <p><code>name</code>：key名；<code>value</code>：元素；<code>amount</code>：增长的<code>score</code>值</p>
                  <p><code>redis.zincrby(&#39;grade&#39;, &#39;Bob&#39;, -2)</code></p>
                  <p>键为<code>grade</code>的zset中<code>Bob</code>的<code>score</code>减2</p>
                  <p>98.0，即修改后的值</p>
                  <p><code>zrank(name, value)</code></p>
                  <p>返回键为<code>name</code>的zset中元素的排名，按<code>score</code>从小到大排序，即名次</p>
                  <p><code>name</code>：键名；<code>value</code>：元素值</p>
                  <p><code>redis.zrank(&#39;grade&#39;, &#39;Amy&#39;)</code></p>
                  <p>得到键为<code>grade</code>的zset中<code>Amy</code>的排名</p>
                  <p>1</p>
                  <p><code>zrevrank(name, value)</code></p>
                  <p>返回键为<code>name</code>的zset中元素的倒数排名（按<code>score</code>从大到小排序），即名次</p>
                  <p><code>name</code>：键名；<code>value</code>：元素值</p>
                  <p><code>redis.zrevrank(&#39;grade&#39;, &#39;Amy&#39;)</code></p>
                  <p>得到键为<code>grade</code>的zset中<code>Amy</code>的倒数排名</p>
                  <p>2</p>
                  <p><code>zrevrange(name, start, end, withscores=False)</code></p>
                  <p>返回键为<code>name</code>的zset（按<code>score</code>从大到小排序）中<code>index</code>从<code>start</code>到<code>end</code>的所有元素</p>
                  <p><code>name</code>：键值；<code>start</code>：开始索引；<code>end</code>：结束索引；<code>withscores</code>：是否带<code>score</code></p>
                  <p><code>redis.zrevrange(&#39;grade&#39;, 0, 3)</code></p>
                  <p>返回键为<code>grade</code>的zset中前四名元素</p>
                  <p><code>[b&#39;Bob&#39;, b&#39;Mike&#39;, b&#39;Amy&#39;, b&#39;James&#39;]</code></p>
                  <p><code>zrangebyscore(name, min, max, start=None, num=None, withscores=False)</code></p>
                  <p>返回键为<code>name</code>的zset中<code>score</code>在给定区间的元素</p>
                  <p><code>name</code>：键名；<code>min</code>：最低<code>score</code>；<code>max</code>：最高<code>score</code>； <code>start</code>：起始索引；<code>num</code>：个数；<code>withscores</code>：是否带<code>score</code></p>
                  <p><code>redis.zrangebyscore(&#39;grade&#39;, 80, 95)</code></p>
                  <p>返回键为<code>grade</code>的zset中<code>score</code>在80和95之间的元素</p>
                  <p><code>[b&#39;Bob&#39;, b&#39;Mike&#39;, b&#39;Amy&#39;, b&#39;James&#39;]</code></p>
                  <p><code>zcount(name, min, max)</code></p>
                  <p>返回键为<code>name</code>的zset中<code>score</code>在给定区间的数量</p>
                  <p><code>name</code>：键名；<code>min</code>：最低<code>score</code>；max：最高<code>score</code></p>
                  <p><code>redis.zcount(&#39;grade&#39;, 80, 95)</code></p>
                  <p>返回键为<code>grade</code>的zset中<code>score</code>在80到95的元素个数</p>
                  <p>2</p>
                  <p><code>zcard(name)</code></p>
                  <p>返回键为<code>name</code>的zset的元素个数</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.zcard(&#39;grade&#39;)</code></p>
                  <p>获取键为<code>grade</code>的zset中元素的个数</p>
                  <p>3</p>
                  <p><code>zremrangebyrank(name, min, max)</code></p>
                  <p>删除键为<code>name</code>的zset中排名在给定区间的元素</p>
                  <p><code>name</code>：键名；<code>min</code>：最低位次；<code>max</code>：最高位次</p>
                  <p><code>redis.zremrangebyrank(&#39;grade&#39;, 0, 0)</code></p>
                  <p>删除键为<code>grade</code>的zset中排名第一的元素</p>
                  <p>1，即删除的元素个数</p>
                  <p><code>zremrangebyscore(name, min, max)</code></p>
                  <p>删除键为<code>name</code>的zset中<code>score</code>在给定区间的元素</p>
                  <p><code>name</code>：键名；<code>min</code>：最低<code>score</code>；<code>max</code>：最高<code>score</code></p>
                  <p><code>redis.zremrangebyscore(&#39;grade&#39;, 80, 90)</code></p>
                  <p>删除<code>score</code>在80到90之间的元素</p>
                  <p>1，即删除的元素个数</p>
                  <h2 id="9-散列操作"><a href="#9-散列操作" class="headerlink" title="9. 散列操作"></a>9. 散列操作</h2>
                  <p>Redis还提供了散列表的数据结构，我们可以用<code>name</code>指定一个散列表的名称，表内存储了各个键值对，用法总结如表5-10所示。</p>
                  <p>表5-10 散列操作</p>
                  <p>方法</p>
                  <p>作用</p>
                  <p>参数说明</p>
                  <p>示例</p>
                  <p>示例说明</p>
                  <p>示例结果</p>
                  <p><code>hset(name, key, value)</code></p>
                  <p>向键为<code>name</code>的散列表中添加映射</p>
                  <p><code>name</code>：键名；<code>key</code>：映射键名；<code>value</code>：映射键值</p>
                  <p><code>hset(&#39;price&#39;, &#39;cake&#39;, 5)</code></p>
                  <p>向键为<code>price</code>的散列表中添加映射关系，<code>cake</code>的值为5</p>
                  <p>1，即添加的映射个数</p>
                  <p><code>hsetnx(name, key, value)</code></p>
                  <p>如果映射键名不存在，则向键为<code>name</code>的散列表中添加映射</p>
                  <p><code>name</code>：键名；<code>key</code>：映射键名；<code>value</code>：映射键值</p>
                  <p><code>hsetnx(&#39;price&#39;, &#39;book&#39;, 6)</code></p>
                  <p>向键为<code>price</code>的散列表中添加映射关系，<code>book</code>的值为6</p>
                  <p>1，即添加的映射个数</p>
                  <p><code>hget(name, key)</code></p>
                  <p>返回键为<code>name</code>的散列表中<code>key</code>对应的值</p>
                  <p><code>name</code>：键名；<code>key</code>：映射键名</p>
                  <p><code>redis.hget(&#39;price&#39;, &#39;cake&#39;)</code></p>
                  <p>获取键为<code>price</code>的散列表中键名为<code>cake</code>的值</p>
                  <p>5</p>
                  <p><code>hmget(name, keys, *args)</code></p>
                  <p>返回键为<code>name</code>的散列表中各个键对应的值</p>
                  <p><code>name</code>：键名；<code>keys</code>：映射键名列表</p>
                  <p><code>redis.hmget(&#39;price&#39;, [&#39;apple&#39;, &#39;orange&#39;])</code></p>
                  <p>获取键为<code>price</code>的散列表中<code>apple</code>和<code>orange</code>的值</p>
                  <p><code>[b&#39;3&#39;, b&#39;7&#39;]</code></p>
                  <p><code>hmset(name, mapping)</code></p>
                  <p>向键为<code>name</code>的散列表中批量添加映射</p>
                  <p><code>name</code>：键名；<code>mapping</code>：映射字典</p>
                  <p><code>redis.hmset(&#39;price&#39;, {&#39;banana&#39;: 2, &#39;pear&#39;: 6})</code></p>
                  <p>向键为<code>price</code>的散列表中批量添加映射</p>
                  <p><code>True</code></p>
                  <p><code>hincrby(name, key, amount=1)</code></p>
                  <p>将键为<code>name</code>的散列表中映射的值增加<code>amount</code></p>
                  <p><code>name</code>：键名；<code>key</code>：映射键名；<code>amount</code>：增长量</p>
                  <p><code>redis.hincrby(&#39;price&#39;, &#39;apple&#39;, 3)</code></p>
                  <p><code>key</code>为<code>price</code>的散列表中<code>apple</code>的值增加3</p>
                  <p>6，修改后的值</p>
                  <p><code>hexists(name, key)</code></p>
                  <p>键为<code>name</code>的散列表中是否存在键名为键的映射</p>
                  <p><code>name</code>：键名；<code>key</code>：映射键名</p>
                  <p><code>redis.hexists(&#39;price&#39;, &#39;banana&#39;)</code></p>
                  <p>键为<code>price</code>的散列表中<code>banana</code>的值是否存在</p>
                  <p><code>True</code></p>
                  <p><code>hdel(name, *keys)</code></p>
                  <p>在键为<code>name</code>的散列表中，删除键名为键的映射</p>
                  <p><code>name</code>：键名；<code>keys</code>：映射键名</p>
                  <p><code>redis.hdel(&#39;price&#39;, &#39;banana&#39;)</code></p>
                  <p>从键为<code>price</code>的散列表中删除键名为<code>banana</code>的映射</p>
                  <p><code>True</code></p>
                  <p><code>hlen(name)</code></p>
                  <p>从键为<code>name</code>的散列表中获取映射个数</p>
                  <p><code>name</code>： 键名</p>
                  <p><code>redis.hlen(&#39;price&#39;)</code></p>
                  <p>从键为<code>price</code>的散列表中获取映射个数</p>
                  <p>6</p>
                  <p><code>hkeys(name)</code></p>
                  <p>从键为<code>name</code>的散列表中获取所有映射键名</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.hkeys(&#39;price&#39;)</code></p>
                  <p>从键为<code>price</code>的散列表中获取所有映射键名</p>
                  <p><code>[b&#39;cake&#39;, b&#39;book&#39;, b&#39;banana&#39;, b&#39;pear&#39;]</code></p>
                  <p><code>hvals(name)</code></p>
                  <p>从键为<code>name</code>的散列表中获取所有映射键值</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.hvals(&#39;price&#39;)</code></p>
                  <p>从键为<code>price</code>的散列表中获取所有映射键值</p>
                  <p><code>[b&#39;5&#39;, b&#39;6&#39;, b&#39;2&#39;, b&#39;6&#39;]</code></p>
                  <p><code>hgetall(name)</code></p>
                  <p>从键为<code>name</code>的散列表中获取所有映射键值对</p>
                  <p><code>name</code>：键名</p>
                  <p><code>redis.hgetall(&#39;price&#39;)</code></p>
                  <p>从键为<code>price</code>的散列表中获取所有映射键值对</p>
                  <p><code>{b&#39;cake&#39;: b&#39;5&#39;, b&#39;book&#39;: b&#39;6&#39;, b&#39;orange&#39;: b&#39;7&#39;, b&#39;pear&#39;: b&#39;6&#39;}</code></p>
                  <h2 id="10-RedisDump"><a href="#10-RedisDump" class="headerlink" title="10. RedisDump"></a>10. RedisDump</h2>
                  <p>RedisDump提供了强大的Redis数据的导入和导出功能，现在就来看下它的具体用法。</p>
                  <p>首先，确保已经安装好了RedisDump。</p>
                  <p>RedisDump提供了两个可执行命令：<code>redis-dump</code>用于导出数据，<code>redis-load</code>用于导入数据。</p>
                  <h5 id="redis-dump"><a href="#redis-dump" class="headerlink" title="redis-dump"></a><code>redis-dump</code></h5>
                  <p>首先，可以输入如下命令查看所有可选项：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">redis-dump -h</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Usage</span>: redis-dump [<span class="keyword">global</span> <span class="keyword">options</span>] COMMAND [command <span class="keyword">options</span>] </span><br><span class="line">    -u, <span class="comment">--uri=S                      Redis URI (e.g. redis://hostname[:port])</span></span><br><span class="line">    -d, <span class="comment">--database=S                 Redis database (e.g. -d 15)</span></span><br><span class="line">    -s, <span class="comment">--sleep=S                    Sleep for S seconds after dumping (for debugging)</span></span><br><span class="line">    -c, <span class="comment">--count=S                    Chunk size (default: 10000)</span></span><br><span class="line">    -f, <span class="comment">--filter=S                   Filter selected keys (passed directly to redis' KEYS command)</span></span><br><span class="line">    -O, <span class="comment">--without_optimizations      Disable run time optimizations</span></span><br><span class="line">    -V, <span class="comment">--version                    Display version</span></span><br><span class="line">    -D, <span class="comment">--debug</span></span><br><span class="line">        <span class="comment">--nosafe</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中<code>\-u</code>代表Redis连接字符串，<code>\-d</code>代表数据库代号，<code>\-s</code>代表导出之后的休眠时间，<code>\-c</code>代表分块大小，默认是10000，<code>\-f</code>代表导出时的过滤器，<code>\-O</code>代表禁用运行时优化，<code>\-V</code>用于显示版本，<code>\-D</code>表示开启调试。</p>
                  <p>我们拿本地的Redis做测试，运行在6379端口上，密码为foobared，导出命令如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">redis-dump -u :<span class="symbol">foobared@</span>localhost:<span class="number">6379</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果没有密码的话，可以不加密码前缀，命令如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">redis-dump -u localhost:<span class="number">6379</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后，可以将本地0至15号数据库的所有数据输出出来，例如：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"name"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"James"</span>,<span class="attr">"size"</span>:<span class="number">5</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"name2"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"Durant"</span>,<span class="attr">"size"</span>:<span class="number">6</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"name3"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"Durant"</span>,<span class="attr">"size"</span>:<span class="number">6</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"name4"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"HelloWorld"</span>,<span class="attr">"size"</span>:<span class="number">10</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"name5"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"James"</span>,<span class="attr">"size"</span>:<span class="number">5</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"name6"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"James"</span>,<span class="attr">"size"</span>:<span class="number">5</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"age"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"1"</span>,<span class="attr">"size"</span>:<span class="number">1</span>&#125;</span><br><span class="line">&#123;<span class="attr">"db"</span>:<span class="number">0</span>,<span class="attr">"key"</span>:<span class="string">"age2"</span>,<span class="attr">"ttl"</span>:<span class="number">-1</span>,<span class="attr">"type"</span>:<span class="string">"string"</span>,<span class="attr">"value"</span>:<span class="string">"-5"</span>,<span class="attr">"size"</span>:<span class="number">2</span>&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>每条数据都包含6个字段，其中<code>db</code>即数据库代号，<code>key</code>即键名，<code>ttl</code>即该键值对的有效时间，<code>type</code>即键值类型，<code>value</code>即内容，<code>size</code>即占用空间。</p>
                  <p>如果想要将其输出为JSON行文件，可以使用如下命令：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">redis-dump -u :<span class="symbol">foobared@</span>localhost:<span class="number">6379</span> &gt; ./redis_data.jl</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以成功将Redis的所有数据库的所有数据导出成JSON行文件了。</p>
                  <p>另外，可以使用<code>\-d</code>参数指定某个数据库的导出，例如只导出1号数据库的内容：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">redis-dump -u :<span class="symbol">foobared@</span>localhost:<span class="number">6379</span> -d <span class="number">1</span> &gt; ./redis.data.jl</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果只想导出特定的内容，比如想导出以<code>adsl</code>开头的数据，可以加入<code>\-f</code>参数用来过滤，命令如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">redis-dump -<span class="string">u :</span>foobared<span class="meta">@localhost</span>:<span class="number">6379</span> -f <span class="string">adsl:</span>* &gt; ./redis.data.jl</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中<code>\-f</code>参数即Redis的<code>keys</code>命令的参数，可以写一些过滤规则。</p>
                  <h5 id="redis-load"><a href="#redis-load" class="headerlink" title="redis-load"></a><code>redis-load</code></h5>
                  <p>同样，我们可以首先输入如下命令查看所有可选项：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">redis-load -h</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">redis-<span class="built_in">load</span> <span class="comment">--help</span></span><br><span class="line">  Try: redis-<span class="built_in">load</span> [<span class="built_in">global</span> options] COMMAND [<span class="keyword">command</span> <span class="title">options</span>] </span><br><span class="line">    -u, <span class="comment">--uri=S                      Redis URI (e.g. redis://hostname[:port])</span></span><br><span class="line">    -d, <span class="comment">--database=S                 Redis database (e.g. -d 15)</span></span><br><span class="line">    -s, <span class="comment">--sleep=S                    Sleep for S seconds after dumping (for debugging)</span></span><br><span class="line">    -n, <span class="comment">--no_check_utf8</span></span><br><span class="line">    -V, <span class="comment">--version                    Display version</span></span><br><span class="line">    -D, <span class="comment">--debug</span></span><br><span class="line">        <span class="comment">--nosafe</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中<code>\-u</code>代表Redis连接字符串，<code>\-d</code>代表数据库代号，默认是全部，<code>\-s</code>代表导出之后的休眠时间，<code>\-n</code>代表不检测UTF-8编码，<code>\-V</code>表示显示版本，<code>\-D</code>表示开启调试。</p>
                  <p>我们可以将JSON行文件导入到Redis数据库中：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt; redis_data.json redis-load -u :<span class="symbol">foobared@</span>localhost:<span class="number">6379</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以成功将JSON行文件导入到数据库中了。</p>
                  <p>另外，下面的命令同样可以达到同样的效果：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">cat redis_data.json | redis-load -u :<span class="symbol">foobared@</span>localhost:<span class="number">6379</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>本节中，我们不仅了解了RedisPy对Redis数据库的一些基本操作，还演示了RedisDump对数据的导入导出操作。由于其便捷性和高效性，后面我们会利用Redis实现很多架构，如维护代理池、Cookies池、ADSL拨号代理池、Scrapy-Redis分布式架构等，所以Redis的操作需要好好掌握。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 19:43:41" itemprop="dateCreated datePublished" datetime="2018-01-29T19:43:41+08:00">2018-01-29</time>
                </span>
                <span id="/5587.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.3.2-Redis存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>15k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>14 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5584.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5584.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.3.1-MongoDB存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>MongoDB是由C++语言编写的非关系型数据库，是一个基于分布式文件存储的开源数据库系统，其内容存储形式类似JSON对象，它的字段值可以包含其他文档、数组及文档数组，非常灵活。在这一节中，我们就来看看Python 3下MongoDB的存储操作。</p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>在开始之前，请确保已经安装好了MongoDB并启动了其服务，并且安装好了Python的PyMongo库。如果没有安装，可以参考第1章。</p>
                  <h2 id="2-连接MongoDB"><a href="#2-连接MongoDB" class="headerlink" title="2. 连接MongoDB"></a>2. 连接MongoDB</h2>
                  <p>连接MongoDB时，我们需要使用PyMongo库里面的<code>MongoClient</code>。一般来说，传入MongoDB的IP及端口即可，其中第一个参数为地址<code>host</code>，第二个参数为端口<code>port</code>（如果不给它传递参数，默认是27017）：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymongo</span><br><span class="line">client = pymongo.MongoClient(<span class="attribute">host</span>=<span class="string">'localhost'</span>, <span class="attribute">port</span>=27017)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以创建MongoDB的连接对象了。</p>
                  <p>另外，<code>MongoClient</code>的第一个参数<code>host</code>还可以直接传入MongoDB的连接字符串，它以<code>mongodb</code>开头，例如：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">client</span> = MongoClient(<span class="string">'mongodb://localhost:27017/'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这也可以达到同样的连接效果。</p>
                  <h2 id="3-指定数据库"><a href="#3-指定数据库" class="headerlink" title="3. 指定数据库"></a>3. 指定数据库</h2>
                  <p>MongoDB中可以建立多个数据库，接下来我们需要指定操作哪个数据库。这里我们以test数据库为例来说明，下一步需要在程序中指定要使用的数据库：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">db</span> = client.test</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里调用<code>client</code>的<code>test</code>属性即可返回test数据库。当然，我们也可以这样指定：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">db</span> = client[<span class="string">'test'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这两种方式是等价的。</p>
                  <h2 id="4-指定集合"><a href="#4-指定集合" class="headerlink" title="4. 指定集合"></a>4. 指定集合</h2>
                  <p>MongoDB的每个数据库又包含许多集合（collection），它们类似于关系型数据库中的表。</p>
                  <p>下一步需要指定要操作的集合，这里指定一个集合名称为students。与指定数据库类似，指定集合也有两种方式：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">collection</span> = db.students</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">collection</span> = db[<span class="string">'students'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样我们便声明了一个<code>Collection</code>对象。</p>
                  <h2 id="5-插入数据"><a href="#5-插入数据" class="headerlink" title="5. 插入数据"></a>5. 插入数据</h2>
                  <p>接下来，便可以插入数据了。对于students这个集合，新建一条学生数据，这条数据以字典形式表示：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">student = &#123;</span><br><span class="line">    'id': '<span class="number">20170101</span>',</span><br><span class="line">    'name': 'Jordan',</span><br><span class="line">    'age': <span class="number">20</span>,</span><br><span class="line">    'gender': 'male'</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里指定了学生的学号、姓名、年龄和性别。接下来，直接调用<code>collection</code>的<code>insert()</code>方法即可插入数据，代码如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">collection.insert</span>(<span class="variable">student</span>)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在MongoDB中，每条数据其实都有一个<code>_id</code>属性来唯一标识。如果没有显式指明该属性，MongoDB会自动产生一个<code>ObjectId</code>类型的<code>_id</code>属性。<code>insert()</code>方法会在执行后返回<code>_id</code>值。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight llvm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">5932</span>a<span class="number">68615</span><span class="keyword">c</span><span class="number">2606814</span><span class="keyword">c</span><span class="number">91</span>f<span class="number">3</span>d</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，我们也可以同时插入多条数据，只需要以列表形式传递即可，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">student1 = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20170101'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Jordan'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">20</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'male'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">student2 = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20170202'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Mike'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">21</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'male'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">result = collection.insert([student1, student2])</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回结果是对应的<code>_id</code>的集合：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[ObjectId('<span class="number">5932</span>a<span class="number">8011</span>5c<span class="number">2606</span>a59e8a048'), ObjectId('<span class="number">5932</span>a<span class="number">8011</span>5c<span class="number">2606</span>a59e8a049')]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>实际上，在PyMongo 3.x版本中，官方已经不推荐使用<code>insert()</code>方法了。当然，继续使用也没有什么问题。官方推荐使用<code>insert_one()</code>和<code>insert_many()</code>方法来分别插入单条记录和多条记录，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">student = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20170101'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Jordan'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">20</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'male'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">result = collection.insert_one(student)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.inserted_id)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight llvm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.results.InsertOneResult object at <span class="number">0x10d68b558</span>&gt;</span><br><span class="line"><span class="number">5932</span>ab<span class="number">0</span>f<span class="number">15</span><span class="keyword">c</span><span class="number">2606</span>f<span class="number">0</span><span class="keyword">c</span><span class="number">1</span>cf<span class="number">6</span><span class="keyword">c</span><span class="number">5</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>与<code>insert()</code>方法不同，这次返回的是<code>InsertOneResult</code>对象，我们可以调用其<code>inserted_id</code>属性获取<code>_id</code>。</p>
                  <p>对于<code>insert_many()</code>方法，我们可以将数据以列表形式传递，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">student1 = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20170101'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Jordan'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">20</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'male'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">student2 = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20170202'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Mike'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">21</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'male'</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">result = collection.insert_many([student1, student2])</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.inserted_ids)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.results.InsertManyResult object at <span class="number">0</span>x101dea558&gt;</span><br><span class="line">[ObjectId('<span class="number">5932</span>abf415c<span class="number">260708</span>3d3b2ac'), ObjectId('<span class="number">5932</span>abf415c<span class="number">260708</span>3d3b2ad')]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>该方法返回的类型是<code>InsertManyResult</code>，调用<code>inserted_ids</code>属性可以获取插入数据的<code>_id</code>列表。</p>
                  <h2 id="6-查询"><a href="#6-查询" class="headerlink" title="6. 查询"></a>6. 查询</h2>
                  <p>插入数据后，我们可以利用<code>find_one()</code>或<code>find()</code>方法进行查询，其中<code>find_one()</code>查询得到的是单个结果，<code>find()</code>则返回一个生成器对象。示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">collection.find_one</span>(&#123;<span class="string">'name'</span>: <span class="string">'Mike'</span>&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable"><span class="class">result</span></span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们查询<code>name</code>为<code>Mike</code>的数据，它的返回结果是字典类型，运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;class 'dict'&gt;</span><br><span class="line">&#123;'_id': ObjectId('<span class="number">5932</span>a<span class="number">8011</span>5c<span class="number">2606</span>a59e8a049'), 'id': '<span class="number">20170202</span>', 'name': 'Mike', 'age': <span class="number">21</span>, 'gender': 'male'&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，它多了<code>_id</code>属性，这就是MongoDB在插入过程中自动添加的。</p>
                  <p>此外，我们也可以根据<code>ObjectId</code>来查询，此时需要使用bson库里面的<code>objectid</code>：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">bson.objectid</span> <span class="variable">import</span> <span class="variable">ObjectId</span></span><br><span class="line"></span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">collection.find_one</span>(&#123;<span class="string">'_id'</span>: <span class="function"><span class="title">ObjectId</span>(<span class="string">'593278c115c2602667ec6bae'</span>)&#125;)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其查询结果依然是字典类型，具体如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;'_id': ObjectId('<span class="number">593278</span>c115c<span class="number">260266</span>7ec6bae'), 'id': '<span class="number">20170101</span>', 'name': 'Jordan', 'age': <span class="number">20</span>, 'gender': 'male'&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，如果查询结果不存在，则会返回<code>None</code>。</p>
                  <p>对于多条数据的查询，我们可以使用<code>find()</code>方法。例如，这里查找年龄为20的数据，示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">results</span> = <span class="variable">collection.find</span>(&#123;<span class="string">'age'</span>: <span class="number">20</span>&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">results</span>)</span></span><br><span class="line"><span class="variable">for</span> <span class="variable"><span class="class">result</span></span> <span class="variable"><span class="keyword">in</span></span> <span class="variable">results</span>:</span><br><span class="line">    <span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.cursor.Cursor object at <span class="number">0</span>x1032d5128&gt;</span><br><span class="line">&#123;'_id': ObjectId('<span class="number">593278</span>c115c<span class="number">260266</span>7ec6bae'), 'id': '<span class="number">20170101</span>', 'name': 'Jordan', 'age': <span class="number">20</span>, 'gender': 'male'&#125;</span><br><span class="line">&#123;'_id': ObjectId('<span class="number">593278</span>c815c<span class="number">260267</span>8bb2b8d'), 'id': '<span class="number">20170102</span>', 'name': 'Kevin', 'age': <span class="number">20</span>, 'gender': 'male'&#125;</span><br><span class="line">&#123;'_id': ObjectId('<span class="number">593278</span>d815c<span class="number">260269</span>d<span class="number">7645</span>a8'), 'id': '<span class="number">20170103</span>', 'name': 'Harden', 'age': <span class="number">20</span>, 'gender': 'male'&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回结果是<code>Cursor</code>类型，它相当于一个生成器，我们需要遍历取到所有的结果，其中每个结果都是字典类型。</p>
                  <p>如果要查询年龄大于20的数据，则写法如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">results</span> = collection.find(&#123;<span class="string">'age'</span>: &#123;<span class="string">'$gt'</span>: <span class="number">20</span>&#125;&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里查询的条件键值已经不是单纯的数字了，而是一个字典，其键名为比较符号<code>$gt</code>，意思是大于，键值为20。</p>
                  <p>这里将比较符号归纳为表5-3。</p>
                  <p>表5-3 比较符号</p>
                  <p>符号</p>
                  <p>含义</p>
                  <p>示例</p>
                  <p><code>$lt</code></p>
                  <p>小于</p>
                  <p><code>{&#39;age&#39;: {&#39;$lt&#39;: 20}}</code></p>
                  <p><code>$gt</code></p>
                  <p>大于</p>
                  <p><code>{&#39;age&#39;: {&#39;$gt&#39;: 20}}</code></p>
                  <p><code>$lte</code></p>
                  <p>小于等于</p>
                  <p><code>{&#39;age&#39;: {&#39;$lte&#39;: 20}}</code></p>
                  <p><code>$gte</code></p>
                  <p>大于等于</p>
                  <p><code>{&#39;age&#39;: {&#39;$gte&#39;: 20}}</code></p>
                  <p><code>$ne</code></p>
                  <p>不等于</p>
                  <p><code>{&#39;age&#39;: {&#39;$ne&#39;: 20}}</code></p>
                  <p><code>$in</code></p>
                  <p>在范围内</p>
                  <p><code>{&#39;age&#39;: {&#39;$in&#39;: [20, 23]}}</code></p>
                  <p><code>$nin</code></p>
                  <p>不在范围内</p>
                  <p><code>{&#39;age&#39;: {&#39;$nin&#39;: [20, 23]}}</code></p>
                  <p>另外，还可以进行正则匹配查询。例如，查询名字以M开头的学生数据，示例如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">results</span> = collection.find(&#123;<span class="string">'name'</span>: &#123;<span class="string">'$regex'</span>: <span class="string">'^M.*'</span>&#125;&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用<code>$regex</code>来指定正则匹配，<code>^M.*</code>代表以M开头的正则表达式。</p>
                  <p>这里将一些功能符号再归类为表5-4。</p>
                  <p>表5-4 功能符号</p>
                  <p>符号</p>
                  <p>含义</p>
                  <p>示例</p>
                  <p>示例含义</p>
                  <p><code>$regex</code></p>
                  <p>匹配正则表达式</p>
                  <p><code>{&#39;name&#39;: {&#39;$regex&#39;: &#39;^M.*&#39;}}</code></p>
                  <p><code>name</code>以M开头</p>
                  <p><code>$exists</code></p>
                  <p>属性是否存在</p>
                  <p><code>{&#39;name&#39;: {&#39;$exists&#39;: True}}</code></p>
                  <p><code>name</code>属性存在</p>
                  <p><code>$type</code></p>
                  <p>类型判断</p>
                  <p><code>{&#39;age&#39;: {&#39;$type&#39;: &#39;int&#39;}}</code></p>
                  <p><code>age</code>的类型为<code>int</code></p>
                  <p><code>$mod</code></p>
                  <p>数字模操作</p>
                  <p><code>{&#39;age&#39;: {&#39;$mod&#39;: [5, 0]}}</code></p>
                  <p>年龄模5余0</p>
                  <p><code>$text</code></p>
                  <p>文本查询</p>
                  <p><code>{&#39;$text&#39;: {&#39;$search&#39;: &#39;Mike&#39;}}</code></p>
                  <p><code>text</code>类型的属性中包含<code>Mike</code>字符串</p>
                  <p><code>$where</code></p>
                  <p>高级条件查询</p>
                  <p><code>{&#39;$where&#39;: &#39;obj.fans_count == obj.follows_count&#39;}</code></p>
                  <p>自身粉丝数等于关注数</p>
                  <p>关于这些操作的更详细用法，可以在MongoDB官方文档找到：<a href="https://docs.mongodb.com/manual/reference/operator/query/" target="_blank" rel="noopener">https://docs.mongodb.com/manual/reference/operator/query/</a>。</p>
                  <h2 id="7-计数"><a href="#7-计数" class="headerlink" title="7. 计数"></a>7. 计数</h2>
                  <p>要统计查询结果有多少条数据，可以调用<code>count()</code>方法。比如，统计所有数据条数：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">count</span> = collection.<span class="keyword">find</span>().<span class="keyword">count</span>()</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">count</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者统计符合某个条件的数据：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">count</span> = collection.<span class="keyword">find</span>(&#123;<span class="string">'age'</span>: <span class="number">20</span>&#125;).<span class="keyword">count</span>()</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">count</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果是一个数值，即符合条件的数据条数。</p>
                  <h2 id="8-排序"><a href="#8-排序" class="headerlink" title="8. 排序"></a>8. 排序</h2>
                  <p>排序时，直接调用<code>sort()</code>方法，并在其中传入排序的字段及升降序标志即可。示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">results = collection.<span class="builtin-name">find</span>().sort(<span class="string">'name'</span>, pymongo.ASCENDING)</span><br><span class="line"><span class="builtin-name">print</span>([result[<span class="string">'name'</span>] <span class="keyword">for</span> result <span class="keyword">in</span> results])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'Harden</span>', <span class="symbol">'Jordan</span>', <span class="symbol">'Kevin</span>', <span class="symbol">'Mark</span>', <span class="symbol">'Mike</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们调用<code>pymongo.ASCENDING</code>指定升序。如果要降序排列，可以传入<code>pymongo.DESCENDING</code>。</p>
                  <h2 id="9-偏移"><a href="#9-偏移" class="headerlink" title="9. 偏移"></a>9. 偏移</h2>
                  <p>在某些情况下，我们可能想只取某几个元素，这时可以利用<code>skip()</code>方法偏移几个位置，比如偏移2，就忽略前两个元素，得到第三个及以后的元素：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">results = collection.<span class="builtin-name">find</span>().sort(<span class="string">'name'</span>, pymongo.ASCENDING).skip(2)</span><br><span class="line"><span class="builtin-name">print</span>([result[<span class="string">'name'</span>] <span class="keyword">for</span> result <span class="keyword">in</span> results])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'Kevin</span>', <span class="symbol">'Mark</span>', <span class="symbol">'Mike</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，还可以用<code>limit()</code>方法指定要取的结果个数，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">results = collection.<span class="builtin-name">find</span>().sort(<span class="string">'name'</span>, pymongo.ASCENDING).skip(2).limit(2)</span><br><span class="line"><span class="builtin-name">print</span>([result[<span class="string">'name'</span>] <span class="keyword">for</span> result <span class="keyword">in</span> results])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'Kevin</span>', <span class="symbol">'Mark</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果不使用<code>limit()</code>方法，原本会返回三个结果，加了限制后，会截取两个结果返回。</p>
                  <p>值得注意的是，在数据库数量非常庞大的时候，如千万、亿级别，最好不要使用大的偏移量来查询数据，因为这样很可能导致内存溢出。此时可以使用类似如下操作来查询：</p>
                  <figure class="highlight gradle">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bson.objectid <span class="keyword">import</span> ObjectId</span><br><span class="line">collection.<span class="keyword">find</span>(&#123;<span class="string">'_id'</span>: &#123;<span class="string">'$gt'</span>: ObjectId(<span class="string">'593278c815c2602678bb2b8d'</span>)&#125;&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时需要记录好上次查询的<code>_id</code>。</p>
                  <h2 id="10-更新"><a href="#10-更新" class="headerlink" title="10. 更新"></a>10. 更新</h2>
                  <p>对于数据更新，我们可以使用<code>update()</code>方法，指定更新的条件和更新后的数据即可。例如：</p>
                  <figure class="highlight mel">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">condition</span> = &#123;<span class="string">'name'</span>: <span class="string">'Kevin'</span>&#125;</span><br><span class="line">student = collection.find_one(<span class="keyword">condition</span>)</span><br><span class="line">student[<span class="string">'age'</span>] = <span class="number">25</span></span><br><span class="line">result = collection.update(<span class="keyword">condition</span>, student)</span><br><span class="line"><span class="keyword">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们要更新<code>name</code>为<code>Kevin</code>的数据的年龄：首先指定查询条件，然后将数据查询出来，修改年龄后调用<code>update()</code>方法将原条件和修改后的数据传入。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">&#123;'ok':</span> <span class="number">1</span><span class="string">,</span> <span class="attr">'nModified':</span> <span class="number">1</span><span class="string">,</span> <span class="attr">'n':</span> <span class="number">1</span><span class="string">,</span> <span class="attr">'updatedExisting':</span> <span class="literal">True</span><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回结果是字典形式，<code>ok</code>代表执行成功，<code>nModified</code>代表影响的数据条数。</p>
                  <p>另外，我们也可以使用<code>$set</code>操作符对数据进行更新，代码如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">result</span> = collection.update(condition, &#123;<span class="string">'$set'</span>: student&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样可以只更新<code>student</code>字典内存在的字段。如果原先还有其他字段，则不会更新，也不会删除。而如果不用<code>$set</code>的话，则会把之前的数据全部用<code>student</code>字典替换；如果原本存在其他字段，则会被删除。</p>
                  <p>另外，<code>update()</code>方法其实也是官方不推荐使用的方法。这里也分为<code>update_one()</code>方法和<code>update_many()</code>方法，用法更加严格，它们的第二个参数需要使用<code>$</code>类型操作符作为字典的键名，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">condition = &#123;<span class="string">'name'</span>: <span class="string">'Kevin'</span>&#125;</span><br><span class="line">student = collection.find_one(condition)</span><br><span class="line">student[<span class="string">'age'</span>] = <span class="number">26</span></span><br><span class="line">result = collection.update_one(condition, &#123;<span class="string">'$set'</span>: student&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.matched_count, result.modified_count)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里调用了<code>update_one()</code>方法，第二个参数不能再直接传入修改后的字典，而是需要使用<code>{&#39;$set&#39;: student}</code>这样的形式，其返回结果是<code>UpdateResult</code>类型。然后分别调用<code>matched_count</code>和<code>modified_count</code>属性，可以获得匹配的数据条数和影响的数据条数。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.results.UpdateResult object at <span class="number">0x10d17b678</span>&gt;</span><br><span class="line"><span class="number">1</span> <span class="number">0</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们再看一个例子：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">condition = &#123;<span class="string">'age'</span>: &#123;<span class="string">'$gt'</span>: <span class="number">20</span>&#125;&#125;</span><br><span class="line">result = collection.update_one(condition, &#123;<span class="string">'$inc'</span>: &#123;<span class="string">'age'</span>: <span class="number">1</span>&#125;&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.matched_count, result.modified_count)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里指定查询条件为年龄大于20，然后更新条件为<code>{&#39;$inc&#39;: {&#39;age&#39;: 1}}</code>，也就是年龄加1，执行之后会将第一条符合条件的数据年龄加1。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.results.UpdateResult object at <span class="number">0x10b8874c8</span>&gt;</span><br><span class="line"><span class="number">1</span> <span class="number">1</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到匹配条数为1条，影响条数也为1条。</p>
                  <p>如果调用<code>update_many()</code>方法，则会将所有符合条件的数据都更新，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">condition = &#123;<span class="string">'age'</span>: &#123;<span class="string">'$gt'</span>: <span class="number">20</span>&#125;&#125;</span><br><span class="line">result = collection.update_many(condition, &#123;<span class="string">'$inc'</span>: &#123;<span class="string">'age'</span>: <span class="number">1</span>&#125;&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.matched_count, result.modified_count)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时匹配条数就不再为1条了，运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.results.UpdateResult object at <span class="number">0x10c6384c8</span>&gt;</span><br><span class="line"><span class="number">3</span> <span class="number">3</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这时所有匹配到的数据都会被更新。</p>
                  <h2 id="11-删除"><a href="#11-删除" class="headerlink" title="11. 删除"></a>11. 删除</h2>
                  <p>删除操作比较简单，直接调用<code>remove()</code>方法指定删除的条件即可，此时符合条件的所有数据均会被删除。示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">collection.remove</span>(&#123;<span class="string">'name'</span>: <span class="string">'Kevin'</span>&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="string">&#123;'ok':</span> <span class="number">1</span><span class="string">,</span> <span class="attr">'n':</span> <span class="number">1</span><span class="string">&#125;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，这里依然存在两个新的推荐方法——<code>delete_one()</code>和<code>delete_many()</code>。示例如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">collection.delete_one</span>(&#123;<span class="string">'name'</span>: <span class="string">'Kevin'</span>&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span></span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span>.deleted_count</span>)</span></span><br><span class="line"><span class="variable"><span class="class">result</span></span> = <span class="variable">collection.delete_many</span>(&#123;<span class="string">'age'</span>: &#123;<span class="string">'$lt'</span>: <span class="number">25</span>&#125;&#125;)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable"><span class="class">result</span>.deleted_count</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;pymongo.results.DeleteResult object at <span class="number">0x10e6ba4c8</span>&gt;</span><br><span class="line"><span class="number">1</span></span><br><span class="line"><span class="number">4</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p><code>delete_one()</code>即删除第一条符合条件的数据，<code>delete_many()</code>即删除所有符合条件的数据。它们的返回结果都是<code>DeleteResult</code>类型，可以调用<code>deleted_count</code>属性获取删除的数据条数。</p>
                  <h2 id="12-其他操作"><a href="#12-其他操作" class="headerlink" title="12. 其他操作"></a>12. 其他操作</h2>
                  <p>另外，PyMongo还提供了一些组合方法，如<code>find_one_and_delete()</code>、<code>find_one_and_replace()</code>和<code>find_one_and_update()</code>，它们是查找后删除、替换和更新操作，其用法与上述方法基本一致。</p>
                  <p>另外，还可以对索引进行操作，相关方法有<code>create_index()</code>、<code>create_indexes()</code>和<code>drop_index()</code>等。</p>
                  <p>关于PyMongo的详细用法，可以参见官方文档：<a href="http://api.mongodb.com/python/current/api/pymongo/collection.html" target="_blank" rel="noopener">http://api.mongodb.com/python/current/api/pymongo/collection.html</a>。</p>
                  <p>另外，还有对数据库和集合本身等的一些操作，这里不再一一讲解，可以参见官方文档：<a href="http://api.mongodb.com/python/current/api/pymongo/" target="_blank" rel="noopener">http://api.mongodb.com/python/current/api/pymongo/</a>。</p>
                  <p>本节讲解了使用PyMongo操作MongoDB进行数据增删改查的方法，后面我们会在实战案例中应用这些操作进行数据存储。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 19:37:47" itemprop="dateCreated datePublished" datetime="2018-01-29T19:37:47+08:00">2018-01-29</time>
                </span>
                <span id="/5584.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.3.1-MongoDB存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>8.9k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>8 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5581.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5581.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.3-非关系型数据库存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>NoSQL，全称Not Only SQL，意为不仅仅是SQL，泛指非关系型数据库。NoSQL是基于键值对的，而且不需要经过SQL层的解析，数据之间没有耦合性，性能非常高。</p>
                  <p>非关系型数据库又可细分如下。</p>
                  <ul>
                    <li><strong>键值存储数据库</strong>：代表有Redis、Voldemort和Oracle BDB等。</li>
                    <li><strong>列存储数据库</strong>：代表有Cassandra、HBase和Riak等。</li>
                    <li><strong>文档型数据库</strong>：代表有CouchDB和MongoDB等。</li>
                    <li><strong>图形数据库</strong>：代表有Neo4J、InfoGrid和Infinite Graph等。</li>
                  </ul>
                  <p>对于爬虫的数据存储来说，一条数据可能存在某些字段提取失败而缺失的情况，而且数据可能随时调整。另外，数据之间还存在嵌套关系。如果使用关系型数据库存储，一是需要提前建表，二是如果存在数据嵌套关系的话，需要进行序列化操作才可以存储，这非常不方便。如果用了非关系型数据库，就可以避免一些麻烦，更简单高效。</p>
                  <p>本节中，我们主要介绍MongoDB和Redis的数据存储操作。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 19:35:58" itemprop="dateCreated datePublished" datetime="2018-01-29T19:35:58+08:00">2018-01-29</time>
                </span>
                <span id="/5581.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.3-非关系型数据库存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>412</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5578.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5578.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.2.1-MySQL存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在Python 2中，连接MySQL的库大多是使用MySQLdb，但是此库的官方并不支持Python 3，所以这里推荐使用的库是PyMySQL。</p>
                  <p>本节中，我们就来讲解使用PyMySQL操作MySQL数据库的方法。</p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>在开始之前，请确保已经安装好了MySQL数据库并保证它能正常运行，而且需要安装好PyMySQL库。如果没有安装，可以参考第1章。</p>
                  <h2 id="2-连接数据库"><a href="#2-连接数据库" class="headerlink" title="2. 连接数据库"></a>2. 连接数据库</h2>
                  <p>这里，首先尝试连接一下数据库。假设当前的MySQL运行在本地，用户名为root，密码为123456，运行端口为3306。这里利用PyMySQL先连接MySQL，然后创建一个新的数据库，名字叫作spiders，代码如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> pymysql</span><br><span class="line"></span><br><span class="line">db = pymysql.<span class="keyword">connect</span>(host=<span class="string">'localhost'</span>,<span class="keyword">user</span>=<span class="string">'root'</span>, <span class="keyword">password</span>=<span class="string">'123456'</span>, port=<span class="number">3306</span>)</span><br><span class="line"><span class="keyword">cursor</span> = db.<span class="keyword">cursor</span>()</span><br><span class="line"><span class="keyword">cursor</span>.<span class="keyword">execute</span>(<span class="string">'SELECT VERSION()'</span>)</span><br><span class="line">data = <span class="keyword">cursor</span>.fetchone()</span><br><span class="line">print(<span class="string">'Database version:'</span>, data)</span><br><span class="line"><span class="keyword">cursor</span>.<span class="keyword">execute</span>("CREATE DATABASE spiders DEFAULT CHARACTER SET utf8")</span><br><span class="line">db.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Database</span> <span class="keyword">version</span>: (<span class="string">'5.6.22'</span>,)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里通过PyMySQL的<code>connect()</code>方法声明一个MySQL连接对象<code>db</code>，此时需要传入MySQL运行的<code>host</code>（即IP）。由于MySQL在本地运行，所以传入的是<code>localhost</code>。如果MySQL在远程运行，则传入其公网IP地址。后续的参数<code>user</code>即用户名，<code>password</code>即密码，<code>port</code>即端口（默认为<code>3306</code>）。</p>
                  <p>连接成功后，需要再调用<code>cursor()</code>方法获得MySQL的操作游标，利用游标来执行SQL语句。这里我们执行了两句SQL，直接用<code>execute()</code>方法执行即可。第一句SQL用于获得MySQL的当前版本，然后调用<code>fetchone()</code>方法获得第一条数据，也就得到了版本号。第二句SQL执行创建数据库的操作，数据库名叫作spiders，默认编码为UTF-8。由于该语句不是查询语句，所以直接执行后就成功创建了数据库spiders。接着，再利用这个数据库进行后续的操作。</p>
                  <h2 id="3-创建表"><a href="#3-创建表" class="headerlink" title="3. 创建表"></a>3. 创建表</h2>
                  <p>一般来说，创建数据库的操作只需要执行一次就好了。当然，我们也可以手动创建数据库。以后，我们的操作都在spiders数据库上执行。</p>
                  <p>创建数据库后，在连接时需要额外指定一个参数<code>db</code>。</p>
                  <p>接下来，新创建一个数据表students，此时执行创建表的SQL语句即可。这里指定3个字段，结构如表5-1所示。</p>
                  <p>表5-1 数据表students</p>
                  <p>字段名</p>
                  <p>含义</p>
                  <p>类型</p>
                  <p><code>id</code></p>
                  <p>学号</p>
                  <p><code>varchar</code></p>
                  <p><code>name</code></p>
                  <p>姓名</p>
                  <p><code>varchar</code></p>
                  <p><code>age</code></p>
                  <p>年龄</p>
                  <p><code>int</code></p>
                  <p>创建该表的示例代码如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> pymysql</span><br><span class="line"></span><br><span class="line">db = pymysql.<span class="keyword">connect</span>(host=<span class="string">'localhost'</span>, <span class="keyword">user</span>=<span class="string">'root'</span>, <span class="keyword">password</span>=<span class="string">'123456'</span>, port=<span class="number">3306</span>, db=<span class="string">'spiders'</span>)</span><br><span class="line"><span class="keyword">cursor</span> = db.<span class="keyword">cursor</span>()</span><br><span class="line"><span class="keyword">sql</span> = <span class="string">'CREATE TABLE IF NOT EXISTS students (id VARCHAR(255) NOT NULL, name VARCHAR(255) NOT NULL, age INT NOT NULL, PRIMARY KEY (id))'</span></span><br><span class="line"><span class="keyword">cursor</span>.<span class="keyword">execute</span>(<span class="keyword">sql</span>)</span><br><span class="line">db.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行之后，我们便创建了一个名为students的数据表。</p>
                  <p>当然，为了演示，这里只指定了最简单的几个字段。实际上，在爬虫过程中，我们会根据爬取结果设计特定的字段。</p>
                  <h2 id="4-插入数据"><a href="#4-插入数据" class="headerlink" title="4. 插入数据"></a>4. 插入数据</h2>
                  <p>下一步就是向数据库中插入数据了。例如，这里爬取了一个学生信息，学号为20120001，名字为Bob，年龄为20，那么如何将该条数据插入数据库呢？示例代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import pymysql</span><br><span class="line"></span><br><span class="line">id = <span class="string">'20120001'</span></span><br><span class="line">user = <span class="string">'Bob'</span></span><br><span class="line">age = 20</span><br><span class="line"></span><br><span class="line">db = pymysql.connect(<span class="attribute">host</span>=<span class="string">'localhost'</span>, <span class="attribute">user</span>=<span class="string">'root'</span>, <span class="attribute">password</span>=<span class="string">'123456'</span>, <span class="attribute">port</span>=3306, <span class="attribute">db</span>=<span class="string">'spiders'</span>)</span><br><span class="line">cursor = db.cursor()</span><br><span class="line">sql = <span class="string">'INSERT INTO students(id, name, age) values(%s, %s, %s)'</span></span><br><span class="line">try:</span><br><span class="line">    cursor.execute(sql, (id, user, age))</span><br><span class="line">    db.commit()</span><br><span class="line">except:</span><br><span class="line">    db.rollback()</span><br><span class="line">db.close()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先构造了一个SQL语句，其<code>Value</code>值没有用字符串拼接的方式来构造，如：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sql = '<span class="keyword">INSERT</span> <span class="keyword">INTO</span> students(<span class="keyword">id</span>, <span class="keyword">name</span>, age) <span class="keyword">values</span>(<span class="string">' + id + '</span>, <span class="string">' + name + '</span>, <span class="string">' + age + '</span>)<span class="string">'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样的写法烦琐而且不直观，所以我们选择直接用格式化符<code>%s</code>来实现。有几个<code>Value</code>写几个<code>%s</code>，我们只需要在<code>execute()</code>方法的第一个参数传入该SQL语句，<code>Value</code>值用统一的元组传过来就好了。这样的写法既可以避免字符串拼接的麻烦，又可以避免引号冲突的问题。</p>
                  <p>之后值得注意的是，需要执行<code>db</code>对象的<code>commit()</code>方法才可实现数据插入，这个方法才是真正将语句提交到数据库执行的方法。对于数据插入、更新、删除操作，都需要调用该方法才能生效。</p>
                  <p>接下来，我们加了一层异常处理。如果执行失败，则调用<code>rollback()</code>执行数据回滚，相当于什么都没有发生过。</p>
                  <p>这里涉及事务的问题。事务机制可以确保数据的一致性，也就是这件事要么发生了，要么没有发生。比如插入一条数据，不会存在插入一半的情况，要么全部插入，要么都不插入，这就是事务的原子性。另外，事务还有3个属性——一致性、隔离性和持久性。这4个属性通常称为ACID特性，具体如表5-2所示。</p>
                  <p>表5-2 事务的4个属性</p>
                  <p>属性</p>
                  <p>解释</p>
                  <p>原子性（atomicity）</p>
                  <p>事务是一个不可分割的工作单位，事务中包括的诸操作要么都做，要么都不做</p>
                  <p>一致性（consistency）</p>
                  <p>事务必须使数据库从一个一致性状态变到另一个一致性状态。一致性与原子性是密切相关的</p>
                  <p>隔离性（isolation）</p>
                  <p>一个事务的执行不能被其他事务干扰，即一个事务内部的操作及使用的数据对并发的其他事务是隔离的，并发执行的各个事务之间不能互相干扰</p>
                  <p>持久性（durability）</p>
                  <p>持续性也称永久性（permanence），指一个事务一旦提交，它对数据库中数据的改变就应该是永久性的。接下来的其他操作或故障不应该对其有任何影响</p>
                  <p>插入、更新和删除操作都是对数据库进行更改的操作，而更改操作都必须为一个事务，所以这些操作的标准写法就是：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    <span class="built_in">cursor</span>.<span class="keyword">execute</span>(sql)</span><br><span class="line">    db.commit()</span><br><span class="line">excep<span class="variable">t:</span></span><br><span class="line">    db.rollback()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样便可以保证数据的一致性。这里的<code>commit()</code>和<code>rollback()</code>方法就为事务的实现提供了支持。</p>
                  <p>上面数据插入的操作是通过构造SQL语句实现的，但是很明显，这有一个极其不方便的地方，比如突然增加了性别字段<code>gender</code>，此时SQL语句就需要改成：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">INSERT</span> <span class="meta">INTO</span> students(id, name, age, gender) values(<span class="name">%s</span>, <span class="name">%s</span>, <span class="name">%s</span>, <span class="name">%s</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>相应的元组参数则需要改成：</p>
                  <figure class="highlight hy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">(<span class="name"><span class="builtin-name">id</span></span>, name, age, gender)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这显然不是我们想要的。在很多情况下，我们要达到的效果是插入方法无需改动，做成一个通用方法，只需要传入一个动态变化的字典就好了。比如，构造这样一个字典：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">    'id': '<span class="number">20120001</span>',</span><br><span class="line">    'name': 'Bob',</span><br><span class="line">    'age': <span class="number">20</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后SQL语句会根据字典动态构造，元组也动态构造，这样才能实现通用的插入方法。所以，这里我们需要改写一下插入方法：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20120001'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Bob'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">20</span></span><br><span class="line">&#125;</span><br><span class="line">table = <span class="string">'students'</span></span><br><span class="line"><span class="built_in">keys</span> = <span class="string">', '</span>.<span class="keyword">join</span>(data.<span class="built_in">keys</span>())</span><br><span class="line"><span class="built_in">values</span> = <span class="string">', '</span>.<span class="keyword">join</span>([<span class="string">'%s'</span>] * <span class="built_in">len</span>(data))</span><br><span class="line">sql = <span class="string">'INSERT INTO &#123;table&#125;(&#123;keys&#125;) VALUES (&#123;values&#125;)'</span>.format(table=table, <span class="built_in">keys</span>=<span class="built_in">keys</span>, <span class="built_in">values</span>=<span class="built_in">values</span>)</span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">   <span class="keyword">if</span> <span class="built_in">cursor</span>.<span class="keyword">execute</span>(sql, tuple(data.<span class="built_in">values</span>())):</span><br><span class="line">       <span class="keyword">print</span>(<span class="string">'Successful'</span>)</span><br><span class="line">       db.commit()</span><br><span class="line">excep<span class="variable">t:</span></span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Failed'</span>)</span><br><span class="line">    db.rollback()</span><br><span class="line">db.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们传入的数据是字典，并将其定义为<code>data</code>变量。表名也定义成变量<code>table</code>。接下来，就需要构造一个动态的SQL语句了。</p>
                  <p>首先，需要构造插入的字段<code>id</code>、<code>name</code>和<code>age</code>。这里只需要将<code>data</code>的键名拿过来，然后用逗号分隔即可。所以<code>&#39;, &#39;.join(data.keys())</code>的结果就是<code>id, name, age</code>，然后需要构造多个<code>%s</code>当作占位符，有几个字段构造几个即可。比如，这里有三个字段，就需要构造<code>%s, %s, %s</code>。这里首先定义了长度为1的数组<code>[&#39;%s&#39;]</code>，然后用乘法将其扩充为<code>[&#39;%s&#39;, &#39;%s&#39;, &#39;%s&#39;]</code>，再调用<code>join()</code>方法，最终变成<code>%s, %s, %s</code>。最后，我们再利用字符串的<code>format()</code>方法将表名、字段名和占位符构造出来。最终的SQL语句就被动态构造成了：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">INSERT</span> <span class="meta">INTO</span> students(id, name, age) VALUES (<span class="name">%s</span>, <span class="name">%s</span>, <span class="name">%s</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>最后，为<code>execute()</code>方法的第一个参数传入<code>sql</code>变量，第二个参数传入<code>data</code>的键值构造的元组，就可以成功插入数据了。</p>
                  <p>如此以来，我们便实现了传入一个字典来插入数据的方法，不需要再去修改SQL语句和插入操作了。</p>
                  <h2 id="5-更新数据"><a href="#5-更新数据" class="headerlink" title="5. 更新数据"></a>5. 更新数据</h2>
                  <p>数据更新操作实际上也是执行SQL语句，最简单的方式就是构造一个SQL语句，然后执行：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sql = <span class="string">'UPDATE students SET age = %s WHERE name = %s'</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">   <span class="built_in">cursor</span>.<span class="keyword">execute</span>(sql, (<span class="number">25</span>, <span class="string">'Bob'</span>))</span><br><span class="line">   db.commit()</span><br><span class="line">excep<span class="variable">t:</span></span><br><span class="line">   db.rollback()</span><br><span class="line">db.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里同样用占位符的方式构造SQL，然后执行<code>execute()</code>方法，传入元组形式的参数，同样执行<code>commit()</code>方法执行操作。如果要做简单的数据更新的话，完全可以使用此方法。</p>
                  <p>但是在实际的数据抓取过程中，大部分情况下需要插入数据，但是我们关心的是会不会出现重复数据，如果出现了，我们希望更新数据而不是重复保存一次。另外，就像前面所说的动态构造SQL的问题，所以这里可以再实现一种去重的方法，如果数据存在，则更新数据；如果数据不存在，则插入数据。另外，这种做法支持灵活的字典传值。示例如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'id'</span>: <span class="string">'20120001'</span>,</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Bob'</span>,</span><br><span class="line">    <span class="string">'age'</span>: <span class="number">21</span></span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">table = <span class="string">'students'</span></span><br><span class="line"><span class="built_in">keys</span> = <span class="string">', '</span>.<span class="keyword">join</span>(data.<span class="built_in">keys</span>())</span><br><span class="line"><span class="built_in">values</span> = <span class="string">', '</span>.<span class="keyword">join</span>([<span class="string">'%s'</span>] * <span class="built_in">len</span>(data))</span><br><span class="line"></span><br><span class="line">sql = <span class="string">'INSERT INTO &#123;table&#125;(&#123;keys&#125;) VALUES (&#123;values&#125;) ON DUPLICATE KEY UPDATE'</span>.format(table=table, <span class="built_in">keys</span>=<span class="built_in">keys</span>, <span class="built_in">values</span>=<span class="built_in">values</span>)</span><br><span class="line"><span class="keyword">update</span> = <span class="string">','</span>.<span class="keyword">join</span>([<span class="string">" &#123;key&#125; = %s"</span>.format(key=key) <span class="keyword">for</span> key in data])</span><br><span class="line">sql += <span class="keyword">update</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    <span class="keyword">if</span> <span class="built_in">cursor</span>.<span class="keyword">execute</span>(sql, tuple(data.<span class="built_in">values</span>())*<span class="number">2</span>):</span><br><span class="line">        <span class="keyword">print</span>(<span class="string">'Successful'</span>)</span><br><span class="line">        db.commit()</span><br><span class="line">excep<span class="variable">t:</span></span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Failed'</span>)</span><br><span class="line">    db.rollback()</span><br><span class="line">db.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里构造的SQL语句其实是插入语句，但是我们在后面加了<code>ON DUPLICATE KEY UPDATE</code>。这行代码的意思是如果主键已经存在，就执行更新操作。比如，我们传入的数据<code>id</code>仍然为<code>20120001</code>，但是年龄有所变化，由20变成了21，此时这条数据不会被插入，而是直接更新<code>id</code>为<code>20120001</code>的数据。完整的SQL构造出来是这样的：</p>
                  <figure class="highlight sas">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">INSERT</span> <span class="meta">INTO</span> students(id, name, age) VALUES (<span class="name">%s</span>, <span class="name">%s</span>, <span class="name">%s</span>) <span class="meta">ON</span> DUPLICATE <span class="meta">KEY</span> <span class="meta">UPDATE</span> id = <span class="name">%s</span>, name = <span class="name">%s</span>, age = <span class="name">%s</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里就变成了6个<code>%s</code>。所以在后面的<code>execute()</code>方法的第二个参数元组就需要乘以2变成原来的2倍。</p>
                  <p>如此一来，我们就可以实现主键不存在便插入数据，存在则更新数据的功能了。</p>
                  <h2 id="6-删除数据"><a href="#6-删除数据" class="headerlink" title="6. 删除数据"></a>6. 删除数据</h2>
                  <p>删除操作相对简单，直接使用<code>DELETE</code>语句即可，只是需要指定要删除的目标表名和删除条件，而且仍然需要使用<code>db</code>的<code>commit()</code>方法才能生效。示例如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">table</span> = <span class="string">'students'</span></span><br><span class="line">condition = <span class="string">'age &gt; 20'</span></span><br><span class="line"></span><br><span class="line">sql = <span class="string">'DELETE FROM  &#123;table&#125; WHERE &#123;condition&#125;'</span>.<span class="built_in">format</span>(<span class="built_in">table</span>=<span class="built_in">table</span>, condition=condition)</span><br><span class="line">try:</span><br><span class="line">    cursor.<span class="built_in">execute</span>(sql)</span><br><span class="line">    db.commit()</span><br><span class="line">except:</span><br><span class="line">    db.rollback()</span><br><span class="line"></span><br><span class="line">db.<span class="built_in">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因为删除条件有多种多样，运算符有大于、小于、等于、<code>LIKE</code>等，条件连接符有<code>AND</code>、<code>OR</code>等，所以不再继续构造复杂的判断条件。这里直接将条件当作字符串来传递，以实现删除操作。</p>
                  <h2 id="7-查询数据"><a href="#7-查询数据" class="headerlink" title="7. 查询数据"></a>7. 查询数据</h2>
                  <p>说完插入、修改和删除等操作，还剩下非常重要的一个操作，那就是查询。查询会用到<code>SELECT</code>语句，示例如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sql = <span class="string">'SELECT * FROM students WHERE age &gt;= 20'</span></span><br><span class="line"></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    <span class="built_in">cursor</span>.<span class="keyword">execute</span>(sql)</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Count:'</span>, <span class="built_in">cursor</span>.rowcount)</span><br><span class="line">    one = <span class="built_in">cursor</span>.fetchone()</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'One:'</span>, one)</span><br><span class="line">    results = <span class="built_in">cursor</span>.fetchall()</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Results:'</span>, results)</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Results Type:'</span>, <span class="built_in">type</span>(results))</span><br><span class="line">    <span class="keyword">for</span> row in result<span class="variable">s:</span></span><br><span class="line">        <span class="keyword">print</span>(row)</span><br><span class="line">excep<span class="variable">t:</span></span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Error'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Count: <span class="number">4</span></span><br><span class="line">One: ('<span class="number">20120001</span>', 'Bob', <span class="number">25</span>)</span><br><span class="line">Results: (('<span class="number">20120011</span>', 'Mary', <span class="number">21</span>), ('<span class="number">20120012</span>', 'Mike', <span class="number">20</span>), ('<span class="number">20120013</span>', 'James', <span class="number">22</span>))</span><br><span class="line">Results Type: &lt;class 'tuple'&gt;</span><br><span class="line">('<span class="number">20120011</span>', 'Mary', <span class="number">21</span>)</span><br><span class="line">('<span class="number">20120012</span>', 'Mike', <span class="number">20</span>)</span><br><span class="line">('<span class="number">20120013</span>', 'James', <span class="number">22</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们构造了一条SQL语句，将年龄20岁及以上的学生查询出来，然后将其传给<code>execute()</code>方法。注意，这里不再需要<code>db</code>的<code>commit()</code>方法。接着，调用<code>cursor</code>的<code>rowcount</code>属性获取查询结果的条数，当前示例中是4条。</p>
                  <p>然后我们调用了<code>fetchone()</code>方法，这个方法可以获取结果的第一条数据，返回结果是元组形式，元组的元素顺序跟字段一一对应，即第一个元素就是第一个字段<code>id</code>，第二个元素就是第二个字段<code>name</code>，以此类推。随后，我们又调用了<code>fetchall()</code>方法，它可以得到结果的所有数据。然后将其结果和类型打印出来，它是二重元组，每个元素都是一条记录，我们将其遍历输出出来。</p>
                  <p>但是这里需要注意一个问题，这里显示的是3条数据而不是4条，<code>fetchall()</code>方法不是获取所有数据吗？这是因为它的内部实现有一个偏移指针用来指向查询结果，最开始偏移指针指向第一条数据，取一次之后，指针偏移到下一条数据，这样再取的话，就会取到下一条数据了。我们最初调用了一次<code>fetchone()</code>方法，这样结果的偏移指针就指向下一条数据，<code>fetchall()</code>方法返回的是偏移指针指向的数据一直到结束的所有数据，所以该方法获取的结果就只剩3个了。</p>
                  <p>此外，我们还可以用<code>while</code>循环加<code>fetchone()</code>方法来获取所有数据，而不是用<code>fetchall()</code>全部一起获取出来。<code>fetchall()</code>会将结果以元组形式全部返回，如果数据量很大，那么占用的开销会非常高。因此，推荐使用如下方法来逐条取数据：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">sql = <span class="string">'SELECT * FROM students WHERE age &gt;= 20'</span></span><br><span class="line"><span class="keyword">try</span>:</span><br><span class="line">    <span class="built_in">cursor</span>.<span class="keyword">execute</span>(sql)</span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Count:'</span>, <span class="built_in">cursor</span>.rowcount)</span><br><span class="line">    row = <span class="built_in">cursor</span>.fetchone()</span><br><span class="line">    <span class="keyword">while</span> ro<span class="variable">w:</span></span><br><span class="line">        <span class="keyword">print</span>(<span class="string">'Row:'</span>, row)</span><br><span class="line">        row = <span class="built_in">cursor</span>.fetchone()</span><br><span class="line">excep<span class="variable">t:</span></span><br><span class="line">    <span class="keyword">print</span>(<span class="string">'Error'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样每循环一次，指针就会偏移一条数据，随用随取，简单高效。</p>
                  <p>本节中，我们介绍了如何使用PyMySQL操作MySQL数据库以及一些SQL语句的构造方法，后面会在实战案例中应用这些操作来存储数据。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 19:29:25" itemprop="dateCreated datePublished" datetime="2018-01-29T19:29:25+08:00">2018-01-29</time>
                </span>
                <span id="/5578.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.2.1-MySQL存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>7.3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>7 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5575.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5575.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.2-关系型数据库存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>关系型数据库是基于关系模型的数据库，而关系模型是通过二维表来保存的，所以它的存储方式就是行列组成的表，每一列是一个字段，每一行是一条记录。表可以看作某个实体的集合，而实体之间存在联系，这就需要表与表之间的关联关系来体现，如主键外键的关联关系。多个表组成一个数据库，也就是关系型数据库。</p>
                  <p>关系型数据库有多种，如SQLite、MySQL、Oracle、SQL Server、DB2等。</p>
                  <p>本节中，我们主要介绍Python 3下MySQL的存储。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 19:25:55" itemprop="dateCreated datePublished" datetime="2018-01-29T19:25:55+08:00">2018-01-29</time>
                </span>
                <span id="/5575.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.2-关系型数据库存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>215</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5571.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5571.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.1.3-CSV文件存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>CSV，全称为Comma-Separated Values，中文可以叫作逗号分隔值或字符分隔值，其文件以纯文本形式存储表格数据。该文件是一个字符序列，可以由任意数目的记录组成，记录间以某种换行符分隔。每条记录由字段组成，字段间的分隔符是其他字符或字符串，最常见的是逗号或制表符。不过所有记录都有完全相同的字段序列，相当于一个结构化表的纯文本形式。它比Excel文件更加简介，XLS文本是电子表格，它包含了文本、数值、公式和格式等内容，而CSV中不包含这些内容，就是特定字符分隔的纯文本，结构简单清晰。所以，有时候用CSV来保存数据是比较方便的。本节中，我们来讲解Python读取和写入CSV文件的过程。</p>
                  <h2 id="1-写入"><a href="#1-写入" class="headerlink" title="1. 写入"></a>1. 写入</h2>
                  <p>这里先看一个最简单的例子：</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import csv</span><br><span class="line"></span><br><span class="line">with open(<span class="string">'data.csv'</span>, <span class="string">'w'</span>) as csvfile:</span><br><span class="line">    writer = csv.writer(csvfile)</span><br><span class="line">    writer.writerow([<span class="string">'id'</span>, <span class="string">'name'</span>, <span class="string">'age'</span>])</span><br><span class="line">    writer.writerow([<span class="string">'10001'</span>, <span class="string">'Mike'</span>, <span class="number">20</span>])</span><br><span class="line">    writer.writerow([<span class="string">'10002'</span>, <span class="string">'Bob'</span>, <span class="number">22</span>])</span><br><span class="line">    writer.writerow([<span class="string">'10003'</span>, <span class="string">'Jordan'</span>, <span class="number">21</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，打开data.csv文件，然后指定打开的模式为<code>w</code>（即写入），获得文件句柄，随后调用csv库的<code>writer()</code>方法初始化写入对象，传入该句柄，然后调用<code>writerow()</code>方法传入每行的数据即可完成写入。</p>
                  <p>运行结束后，会生成一个名为data.csv的文件，此时数据就成功写入了。直接以文本形式打开的话，其内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">id,name,age</span><br><span class="line"><span class="number">10001</span>,Mike,<span class="number">20</span></span><br><span class="line"><span class="number">10002</span>,Bob,<span class="number">22</span></span><br><span class="line"><span class="number">10003</span>,Jordan,<span class="number">21</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，写入的文本默认以逗号分隔，调用一次<code>writerow()</code>方法即可写入一行数据。用Excel打开的结果如图5-6所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/5-6.jpg" alt="">图5-6 打开结果</p>
                  <p>如果想修改列与列之间的分隔符，可以传入<code>delimiter</code>参数，其代码如下：</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import csv</span><br><span class="line"></span><br><span class="line">with open(<span class="string">'data.csv'</span>, <span class="string">'w'</span>) as csvfile:</span><br><span class="line">    writer = csv.writer(csvfile, delimiter=<span class="string">' '</span>)</span><br><span class="line">    writer.writerow([<span class="string">'id'</span>, <span class="string">'name'</span>, <span class="string">'age'</span>])</span><br><span class="line">    writer.writerow([<span class="string">'10001'</span>, <span class="string">'Mike'</span>, <span class="number">20</span>])</span><br><span class="line">    writer.writerow([<span class="string">'10002'</span>, <span class="string">'Bob'</span>, <span class="number">22</span>])</span><br><span class="line">    writer.writerow([<span class="string">'10003'</span>, <span class="string">'Jordan'</span>, <span class="number">21</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里在初始化写入对象时传入<code>delimiter</code>为空格，此时输出结果的每一列就是以空格分隔了，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">id name age</span><br><span class="line"><span class="number">10001</span> Mike <span class="number">20</span></span><br><span class="line"><span class="number">10002</span> Bob <span class="number">22</span></span><br><span class="line"><span class="number">10003</span> Jordan <span class="number">21</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，我们也可以调用<code>writerows()</code>方法同时写入多行，此时参数就需要为二维列表，例如：</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import csv</span><br><span class="line"></span><br><span class="line">with open(<span class="string">'data.csv'</span>, <span class="string">'w'</span>) as csvfile:</span><br><span class="line">    writer = csv.writer(csvfile)</span><br><span class="line">    writer.writerow([<span class="string">'id'</span>, <span class="string">'name'</span>, <span class="string">'age'</span>])</span><br><span class="line">    writer.writerows([[<span class="string">'10001'</span>, <span class="string">'Mike'</span>, <span class="number">20</span>], [<span class="string">'10002'</span>, <span class="string">'Bob'</span>, <span class="number">22</span>], [<span class="string">'10003'</span>, <span class="string">'Jordan'</span>, <span class="number">21</span>]])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>输出效果是相同的，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">id,name,age</span><br><span class="line"><span class="number">10001</span>,Mike,<span class="number">20</span></span><br><span class="line"><span class="number">10002</span>,Bob,<span class="number">22</span></span><br><span class="line"><span class="number">10003</span>,Jordan,<span class="number">21</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但是一般情况下，爬虫爬取的都是结构化数据，我们一般会用字典来表示。在csv库中也提供了字典的写入方式，示例如下：</p>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"></span><br><span class="line"><span class="keyword">with</span> open(<span class="string">'data.csv'</span>, <span class="string">'w'</span>) <span class="keyword">as</span> csvfile:</span><br><span class="line">    fieldnames = [<span class="string">'id'</span>, <span class="string">'name'</span>, <span class="string">'age'</span>]</span><br><span class="line">    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)</span><br><span class="line">    writer.writeheader()</span><br><span class="line">    writer.writerow(&#123;<span class="string">'id'</span>: <span class="string">'10001'</span>, <span class="string">'name'</span>: <span class="string">'Mike'</span>, <span class="string">'age'</span>: <span class="number">20</span>&#125;)</span><br><span class="line">    writer.writerow(&#123;<span class="string">'id'</span>: <span class="string">'10002'</span>, <span class="string">'name'</span>: <span class="string">'Bob'</span>, <span class="string">'age'</span>: <span class="number">22</span>&#125;)</span><br><span class="line">    writer.writerow(&#123;<span class="string">'id'</span>: <span class="string">'10003'</span>, <span class="string">'name'</span>: <span class="string">'Jordan'</span>, <span class="string">'age'</span>: <span class="number">21</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里先定义3个字段，用<code>fieldnames</code>表示，然后将其传给<code>DictWriter</code>来初始化一个字典写入对象，接着可以调用<code>writeheader()</code>方法先写入头信息，然后再调用<code>writerow()</code>方法传入相应字典即可。最终写入的结果是完全相同的，内容如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">id,name,age</span><br><span class="line"><span class="number">10001</span>,Mike,<span class="number">20</span></span><br><span class="line"><span class="number">10002</span>,Bob,<span class="number">22</span></span><br><span class="line"><span class="number">10003</span>,Jordan,<span class="number">21</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以完成字典到CSV文件的写入了。</p>
                  <p>另外，如果想追加写入的话，可以修改文件的打开模式，即将<code>open()</code>函数的第二个参数改成<code>a</code>，代码如下：</p>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> csv</span><br><span class="line"></span><br><span class="line"><span class="keyword">with</span> open(<span class="string">'data.csv'</span>, <span class="string">'a'</span>) <span class="keyword">as</span> csvfile:</span><br><span class="line">    fieldnames = [<span class="string">'id'</span>, <span class="string">'name'</span>, <span class="string">'age'</span>]</span><br><span class="line">    writer = csv.DictWriter(csvfile, fieldnames=fieldnames)</span><br><span class="line">    writer.writerow(&#123;<span class="string">'id'</span>: <span class="string">'10004'</span>, <span class="string">'name'</span>: <span class="string">'Durant'</span>, <span class="string">'age'</span>: <span class="number">22</span>&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样在上面的基础上再执行这段代码，文件内容便会变成：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">id,name,age</span><br><span class="line"><span class="number">10001</span>,Mike,<span class="number">20</span></span><br><span class="line"><span class="number">10002</span>,Bob,<span class="number">22</span></span><br><span class="line"><span class="number">10003</span>,Jordan,<span class="number">21</span></span><br><span class="line"><span class="number">10004</span>,Durant,<span class="number">22</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见，数据被追加写入到文件中。</p>
                  <p>如果要写入中文内容的话，可能会遇到字符编码的问题，此时需要给<code>open()</code>参数指定编码格式。比如，这里再写入一行包含中文的数据，代码需要改写如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import csv</span><br><span class="line"></span><br><span class="line">with open(<span class="string">'data.csv'</span>, <span class="string">'a'</span>, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>) as csvfile:</span><br><span class="line">    fieldnames = [<span class="string">'id'</span>, <span class="string">'name'</span>, <span class="string">'age'</span>]</span><br><span class="line">    writer = csv.DictWriter(csvfile, <span class="attribute">fieldnames</span>=fieldnames)</span><br><span class="line">    writer.writerow(&#123;<span class="string">'id'</span>: <span class="string">'10005'</span>, <span class="string">'name'</span>: <span class="string">'王伟'</span>, <span class="string">'age'</span>: 22&#125;)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里需要给<code>open()</code>函数指定编码，否则可能发生编码错误。</p>
                  <p>另外，如果接触过pandas等库的话，可以调用<code>DataFrame</code>对象的<code>to_csv()</code>方法来将数据写入CSV文件中。</p>
                  <h2 id="2-读取"><a href="#2-读取" class="headerlink" title="2. 读取"></a>2. 读取</h2>
                  <p>我们同样可以使用csv库来读取CSV文件。例如，将刚才写入的文件内容读取出来，相关代码如下：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import csv</span><br><span class="line"></span><br><span class="line"><span class="keyword">with</span> <span class="keyword">open</span>(<span class="string">'data.csv'</span>, <span class="string">'r'</span>, <span class="keyword">encoding</span>=<span class="string">'utf-8'</span>) <span class="keyword">as</span> csvfile:</span><br><span class="line">    reader = csv.reader(csvfile)</span><br><span class="line">    <span class="keyword">for</span> <span class="keyword">row</span> <span class="keyword">in</span> reader:</span><br><span class="line">        print(<span class="keyword">row</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'id</span>', <span class="symbol">'name</span>', <span class="symbol">'age</span>']</span><br><span class="line">[<span class="symbol">'10001</span>', <span class="symbol">'Mike</span>', <span class="symbol">'20</span>']</span><br><span class="line">[<span class="symbol">'10002</span>', <span class="symbol">'Bob</span>', <span class="symbol">'22</span>']</span><br><span class="line">[<span class="symbol">'10003</span>', <span class="symbol">'Jordan</span>', <span class="symbol">'21</span>']</span><br><span class="line">[<span class="symbol">'10004</span>', <span class="symbol">'Durant</span>', <span class="symbol">'22</span>']</span><br><span class="line">[<span class="symbol">'10005</span>', <span class="symbol">'王伟</span>', <span class="symbol">'22</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们构造的是<code>Reader</code>对象，通过遍历输出了每行的内容，每一行都是一个列表形式。注意，如果CSV文件中包含中文的话，还需要指定文件编码。</p>
                  <p>另外，如果接触过pandas的话，可以利用<code>read_csv()</code>方法将数据从CSV中读取出来，例如：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> pandas  <span class="keyword">as</span> pd</span><br><span class="line"></span><br><span class="line"><span class="title">df</span> = pd.read_csv('<span class="class"><span class="keyword">data</span>.csv')</span></span><br><span class="line"><span class="title">print</span>(df)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">      id    name  age</span><br><span class="line"><span class="number">0</span>  <span class="number">10001</span>    Mike   <span class="number">20</span></span><br><span class="line"><span class="number">1</span>  <span class="number">10002</span>     Bob   <span class="number">22</span></span><br><span class="line"><span class="number">2</span>  <span class="number">10003</span>  Jordan   <span class="number">21</span></span><br><span class="line"><span class="number">3</span>  <span class="number">10004</span>  Durant   <span class="number">22</span></span><br><span class="line"><span class="number">4</span>  <span class="number">10005</span>      王伟   <span class="number">22</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在做数据分析的时候，此种方法用得比较多，也是一种比较方便地读取CSV文件的方法。</p>
                  <p>本节中，我们了解了CSV文件的写入和读取方式。这也是一种常用的数据存储方式，需要熟练掌握。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 16:59:52" itemprop="dateCreated datePublished" datetime="2018-01-29T16:59:52+08:00">2018-01-29</time>
                </span>
                <span id="/5571.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.1.3-CSV文件存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.5k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5564.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5564.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.1.2-JSON文件存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>JSON，全称为JavaScript Object Notation, 也就是JavaScript对象标记，它通过对象和数组的组合来表示数据，构造简洁但是结构化程度非常高，是一种轻量级的数据交换格式。本节中，我们就来了解如何利用Python保存数据到JSON文件。</p>
                  <h2 id="1-对象和数组"><a href="#1-对象和数组" class="headerlink" title="1. 对象和数组"></a>1. 对象和数组</h2>
                  <p>在JavaScript语言中，一切都是对象。因此，任何支持的类型都可以通过JSON来表示，例如字符串、数字、对象、数组等，但是对象和数组是比较特殊且常用的两种类型，下面简要介绍一下它们。</p>
                  <ul>
                    <li><strong>对象</strong>：它在JavaScript中是使用花括号<code>{}</code>包裹起来的内容，数据结构为<code>{key1：value1, key2：value2, ...}</code>的键值对结构。在面向对象的语言中，<code>key</code>为对象的属性，<code>value</code>为对应的值。键名可以使用整数和字符串来表示。值的类型可以是任意类型。</li>
                    <li><strong>数组</strong>：数组在JavaScript中是方括号<code>[]</code>包裹起来的内容，数据结构为<code>[&quot;java&quot;, &quot;javascript&quot;, &quot;vb&quot;, ...]</code>的索引结构。在JavaScript中，数组是一种比较特殊的数据类型，它也可以像对象那样使用键值对，但还是索引用得多。同样，值的类型可以是任意类型。</li>
                  </ul>
                  <p>所以，一个JSON对象可以写为如下形式：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&#123;</span><br><span class="line">    <span class="attr">"name"</span>: <span class="string">"Bob"</span>,</span><br><span class="line">    <span class="attr">"gender"</span>: <span class="string">"male"</span>,</span><br><span class="line">    <span class="attr">"birthday"</span>: <span class="string">"1992-10-18"</span></span><br><span class="line">&#125;, &#123;</span><br><span class="line">     <span class="attr">"name"</span>: <span class="string">"Selina"</span>,</span><br><span class="line">    <span class="attr">"gender"</span>: <span class="string">"female"</span>,</span><br><span class="line">    <span class="attr">"birthday"</span>: <span class="string">"1995-10-18"</span></span><br><span class="line">&#125;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>由中括号包围的就相当于列表类型，列表中的每个元素可以是任意类型，这个示例中它是字典类型，由大括号包围。</p>
                  <p>JSON可以由以上两种形式自由组合而成，可以无限次嵌套，结构清晰，是数据交换的极佳方式。</p>
                  <h2 id="2-读取JSON"><a href="#2-读取JSON" class="headerlink" title="2. 读取JSON"></a>2. 读取JSON</h2>
                  <p>Python为我们提供了简单易用的库来实现JSON文件的读写操作，我们可以调用库的<code>loads()</code>方法将JSON文本字符串转为JSON对象，可以通过<code>dumps()</code>方法将JSON对象转为文本字符串。</p>
                  <p>例如，这里有一段JSON形式的字符串，它是<code>str</code>类型，我们用Python将其转换为可操作的数据结构，如列表或字典：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"><span class="title">str</span> = '''</span><br><span class="line">[&#123;</span><br><span class="line">    <span class="string">"name"</span>: <span class="string">"Bob"</span>,</span><br><span class="line">    <span class="string">"gender"</span>: <span class="string">"male"</span>,</span><br><span class="line">    <span class="string">"birthday"</span>: <span class="string">"1992-10-18"</span></span><br><span class="line">&#125;, &#123;</span><br><span class="line">    <span class="string">"name"</span>: <span class="string">"Selina"</span>,</span><br><span class="line">    <span class="string">"gender"</span>: <span class="string">"female"</span>,</span><br><span class="line">    <span class="string">"birthday"</span>: <span class="string">"1995-10-18"</span></span><br><span class="line">&#125;]</span><br><span class="line">'''</span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">str</span>))</span></span><br><span class="line"><span class="class"><span class="keyword">data</span> = json.loads(<span class="title">str</span>)</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">data</span>)</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">data</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;class 'str'&gt;</span><br><span class="line">[&#123;'name': 'Bob', 'gender': 'male', 'birthday': '<span class="number">1992-10-18</span>'&#125;, &#123;'name': 'Selina', 'gender': 'female', 'birthday': '<span class="number">1995-10-18</span>'&#125;]</span><br><span class="line">&lt;class 'list'&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用<code>loads()</code>方法将字符串转为JSON对象。由于最外层是中括号，所以最终的类型是列表类型。</p>
                  <p>这样一来，我们就可以用索引来获取对应的内容了。例如，如果想取第一个元素里的<code>name</code>属性，就可以使用如下方式：</p>
                  <figure class="highlight markdown">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">data[<span class="string">0</span>][<span class="symbol">'name'</span>]</span><br><span class="line">data[0].get('name')</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>得到的结果都是：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">Bob</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过中括号加0索引，可以得到第一个字典元素，然后再调用其键名即可得到相应的键值。获取键值时有两种方式，一种是中括号加键名，另一种是通过<code>get()</code>方法传入键名。这里推荐使用<code>get()</code>方法，这样如果键名不存在，则不会报错，会返回<code>None</code>。另外，<code>get()</code>方法还可以传入第二个参数（即默认值），示例如下：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="class"><span class="keyword">data</span>[0].get('<span class="title">age'</span>)</span></span><br><span class="line"><span class="class"><span class="keyword">data</span>[0].get('<span class="title">age'</span>, 25)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">None</span><br><span class="line"><span class="number">25</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们尝试获取年龄<code>age</code>，其实在原字典中该键名不存在，此时默认会返回<code>None</code>。如果传入第二个参数（即默认值），那么在不存在的情况下返回该默认值。</p>
                  <p>值得注意的是，JSON的数据需要用双引号来包围，不能使用单引号。例如，若使用如下形式表示，则会出现错误：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> <span class="type">json</span></span><br><span class="line"></span><br><span class="line">str = <span class="string">'''</span></span><br><span class="line"><span class="string">[&#123;</span></span><br><span class="line"><span class="string">    '</span><span class="type">name</span><span class="string">': '</span>Bob<span class="string">',</span></span><br><span class="line"><span class="string">    '</span>gender<span class="string">': '</span>mal<span class="string">e',</span></span><br><span class="line"><span class="string">    '</span>birthday<span class="string">': '</span><span class="number">1992</span><span class="number">-10</span><span class="number">-18</span><span class="string">'</span></span><br><span class="line"><span class="string">&#125;]</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line">data = <span class="type">json</span>.loads(str)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="type">json</span>.decoder.JSONDecodeError: Expecting property <span class="type">name</span> enclosed <span class="keyword">in</span> <span class="type">double</span> quotes: <span class="type">line</span> <span class="number">3</span> <span class="keyword">column</span> <span class="number">5</span> (<span class="type">char</span> <span class="number">8</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里会出现JSON解析错误的提示。这是因为这里数据用单引号来包围，请千万注意JSON字符串的表示需要用双引号，否则<code>loads()</code>方法会解析失败。</p>
                  <p>如果从JSON文本中读取内容，例如这里有一个data.文本文件，其内容是刚才定义的JSON字符串，我们可以先将文本文件内容读出，然后再利用<code>loads()</code>方法转化：</p>
                  <figure class="highlight sql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import json</span><br><span class="line"></span><br><span class="line"><span class="keyword">with</span> <span class="keyword">open</span>(<span class="string">'data.json'</span>, <span class="string">'r'</span>) <span class="keyword">as</span> <span class="keyword">file</span>:</span><br><span class="line">    <span class="keyword">str</span> = file.read()</span><br><span class="line">    <span class="keyword">data</span> = json.loads(<span class="keyword">str</span>)</span><br><span class="line">    print(<span class="keyword">data</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&#123;<span class="symbol">'name</span><span class="symbol">':</span> <span class="symbol">'Bob</span>', <span class="symbol">'gender</span><span class="symbol">':</span> <span class="symbol">'male</span>', <span class="symbol">'birthday</span><span class="symbol">':</span> <span class="symbol">'1992-10-18</span>'&#125;, &#123;<span class="symbol">'name</span><span class="symbol">':</span> <span class="symbol">'Selina</span>', <span class="symbol">'gender</span><span class="symbol">':</span> <span class="symbol">'female</span>', <span class="symbol">'birthday</span><span class="symbol">':</span> <span class="symbol">'1995-10-18</span>'&#125;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="3-输出JSON"><a href="#3-输出JSON" class="headerlink" title="3. 输出JSON"></a>3. 输出JSON</h2>
                  <p>另外，我们还可以调用<code>dumps()</code>方法将JSON对象转化为字符串。例如，将上例中的列表重新写入文本：</p>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> json</span><br><span class="line"></span><br><span class="line"><span class="keyword">data</span> = [&#123;</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'Bob'</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'male'</span>,</span><br><span class="line">    <span class="string">'birthday'</span>: <span class="string">'1992-10-18'</span></span><br><span class="line">&#125;]</span><br><span class="line"><span class="keyword">with</span> open(<span class="string">'data.json'</span>, <span class="string">'w'</span>) <span class="keyword">as</span> file:</span><br><span class="line">    file.write(json.dumps(<span class="keyword">data</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>利用<code>dumps()</code>方法，我们可以将JSON对象转为字符串，然后再调用文件的<code>write()</code>方法写入文本，结果如图5-2所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/5-2.jpg" alt="">图5-2 写入结果</p>
                  <p>另外，如果想保存JSON的格式，可以再加一个参数<code>indent</code>，代表缩进字符个数。示例如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">'data.json'</span>, <span class="string">'w'</span>) <span class="keyword">as</span> <span class="built_in">file</span>:</span><br><span class="line">    <span class="built_in">file</span>.<span class="built_in">write</span>(json.dumps(data, indent=<span class="number">2</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此时写入结果如图5-3所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/5-3.jpg" alt="">图5-3 写入结果</p>
                  <p>这样得到的内容会自动带缩进，格式会更加清晰。</p>
                  <p>另外，如果JSON中包含中文字符，会怎么样呢？例如，我们将之前的JSON的部分值改为中文，再用之前的方法写入到文本：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import json</span><br><span class="line"></span><br><span class="line">data = [&#123;</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'王伟'</span>,</span><br><span class="line">    <span class="string">'gender'</span>: <span class="string">'男'</span>,</span><br><span class="line">    <span class="string">'birthday'</span>: <span class="string">'1992-10-18'</span></span><br><span class="line">&#125;]</span><br><span class="line">with <span class="keyword">open</span>(<span class="string">'data.json'</span>, <span class="string">'w'</span>) <span class="keyword">as</span> <span class="keyword">file</span>:</span><br><span class="line">    <span class="keyword">file</span>.<span class="keyword">write</span>(json.dumps(data, <span class="built_in">indent</span>=<span class="number">2</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>写入结果如图5-4所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/5-4.jpg" alt="">图5-4 写入结果</p>
                  <p>可以看到，中文字符都变成了Unicode字符，这并不是我们想要的结果。</p>
                  <p>为了输出中文，还需要指定参数<code>ensure_ascii</code>为<code>False</code>，另外还要规定文件输出的编码：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">with open(<span class="string">'data.json'</span>, <span class="string">'w'</span>, <span class="attribute">encoding</span>=<span class="string">'utf-8'</span>) as file:</span><br><span class="line">    file.write(json.dumps(data, <span class="attribute">indent</span>=2, <span class="attribute">ensure_ascii</span>=<span class="literal">False</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>写入结果如图5-5所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/5-5.jpg" alt="">图5-5 写入结果</p>
                  <p>可以发现，这样就可以输出JSON为中文了。</p>
                  <p>本节中，我们了解了用Python进行JSON文件读写的方法，后面做数据解析时经常会用到，建议熟练掌握。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 16:54:40" itemprop="dateCreated datePublished" datetime="2018-01-29T16:54:40+08:00">2018-01-29</time>
                </span>
                <span id="/5564.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.1.2-JSON文件存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5560.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5560.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.1.1-TXT文本存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>将数据保存到TXT文本的操作非常简单，而且TXT文本几乎兼容任何平台，但是这有个缺点，那就是不利于检索。所以如果对检索和数据结构要求不高，追求方便第一的话，可以采用TXT文本存储。本节中，我们就来看下如何利用Python保存TXT文本文件。</p>
                  <h2 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h2>
                  <p>本节中，我们要保存知乎上“发现”页面的“热门话题”部分，将其问题和答案统一保存成文本形式。</p>
                  <h2 id="2-基本实例"><a href="#2-基本实例" class="headerlink" title="2. 基本实例"></a>2. 基本实例</h2>
                  <p>首先，可以用requests将网页源代码获取下来，然后使用pyquery解析库解析，接下来将提取的标题、回答者、回答保存到文本，代码如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line">from pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line"></span><br><span class="line">url = <span class="string">'https://www.zhihu.com/explore'</span></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36'</span></span><br><span class="line">&#125;</span><br><span class="line">html = requests.<span class="built_in">get</span>(url, headers=headers).text</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="built_in">items</span> = doc(<span class="string">'.explore-tab .feed-item'</span>).<span class="built_in">items</span>()</span><br><span class="line"><span class="keyword">for</span> item in item<span class="variable">s:</span></span><br><span class="line">    question = item.<span class="keyword">find</span>(<span class="string">'h2'</span>).text()</span><br><span class="line">    author = item.<span class="keyword">find</span>(<span class="string">'.author-link-line'</span>).text()</span><br><span class="line">    answer = pq(item.<span class="keyword">find</span>(<span class="string">'.content'</span>).html()).text()</span><br><span class="line">    <span class="keyword">file</span> = <span class="keyword">open</span>(<span class="string">'explore.txt'</span>, <span class="string">'a'</span>, encoding=<span class="string">'utf-8'</span>)</span><br><span class="line">    <span class="keyword">file</span>.<span class="keyword">write</span>(<span class="string">'\n'</span>.<span class="keyword">join</span>([question, author, answer]))</span><br><span class="line">    <span class="keyword">file</span>.<span class="keyword">write</span>(<span class="string">'\n'</span> + <span class="string">'='</span> * <span class="number">50</span> + <span class="string">'\n'</span>)</span><br><span class="line">    <span class="keyword">file</span>.<span class="keyword">close</span>()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里主要是为了演示文件保存的方式，因此requests异常处理部分在此省去。首先，用requests提取知乎的“发现”页面，然后将热门话题的问题、回答者、答案全文提取出来，然后利用Python提供的<code>open()</code>方法打开一个文本文件，获取一个文件操作对象，这里赋值为<code>file</code>，接着利用<code>file</code>对象的<code>write()</code>方法将提取的内容写入文件，最后调用<code>close()</code>方法将其关闭，这样抓取的内容即可成功写入文本中了。</p>
                  <p>运行程序，可以发现在本地生成了一个explore.txt文件，其内容如图5-1所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/5-1.jpg" alt="">图5-1 文件内容</p>
                  <p>这样热门问答的内容就被保存成文本形式了。</p>
                  <p>这里<code>open()</code>方法的第一个参数即要保存的目标文件名称，第二个参数为<code>a</code>，代表以追加方式写入到文本。另外，我们还指定了文件的编码为<code>utf-8</code>。最后，写入完成后，还需要调用<code>close()</code>方法来关闭文件对象。</p>
                  <h2 id="3-打开方式"><a href="#3-打开方式" class="headerlink" title="3. 打开方式"></a>3. 打开方式</h2>
                  <p>在刚才的实例中，<code>open()</code>方法的第二个参数设置成了<code>a</code>，这样在每次写入文本时不会清空源文件，而是在文件末尾写入新的内容，这是一种文件打开方式。关于文件的打开方式，其实还有其他几种，这里简要介绍一下。</p>
                  <ul>
                    <li><strong><code>r</code></strong>：以只读方式打开文件。文件的指针将会放在文件的开头。这是默认模式。</li>
                    <li><strong><code>rb</code></strong>：以二进制只读方式打开一个文件。文件指针将会放在文件的开头。</li>
                    <li><strong><code>r+</code></strong>：以读写方式打开一个文件。文件指针将会放在文件的开头。</li>
                    <li><strong><code>rb+</code></strong>：以二进制读写方式打开一个文件。文件指针将会放在文件的开头。</li>
                    <li><strong><code>w</code></strong>：以写入方式打开一个文件。如果该文件已存在，则将其覆盖。如果该文件不存在，则创建新文件。</li>
                    <li><strong><code>wb</code></strong>：以二进制写入方式打开一个文件。如果该文件已存在，则将其覆盖。如果该文件不存在，则创建新文件。</li>
                    <li><strong><code>w+</code></strong>：以读写方式打开一个文件。如果该文件已存在，则将其覆盖。如果该文件不存在，则创建新文件。</li>
                    <li><strong><code>wb+</code></strong>：以二进制读写格式打开一个文件。如果该文件已存在，则将其覆盖。如果该文件不存在，则创建新文件。</li>
                    <li>
                      <p><strong><code>a</code></strong>：以追加方式打开一个文件。如果该文件已存在，文件指针将会放在文件结尾。也就是说，新的内容将会被写入到已有内容之后。如果该文件不存在，则创建新文件来写入。</p>
                    </li>
                    <li>
                      <p><strong><code>ab</code></strong>：以二进制追加方式打开一个文件。如果该文件已存在，则文件指针将会放在文件结尾。也就是说，新的内容将会被写入到已有内容之后。如果该文件不存在，则创建新文件来写入。</p>
                    </li>
                    <li>
                      <p><strong><code>a+</code></strong>：以读写方式打开一个文件。如果该文件已存在，文件指针将会放在文件的结尾。文件打开时会是追加模式。如果该文件不存在，则创建新文件来读写。</p>
                    </li>
                    <li><strong><code>ab+</code></strong>：以二进制追加方式打开一个文件。如果该文件已存在，则文件指针将会放在文件结尾。如果该文件不存在，则创建新文件用于读写。</li>
                  </ul>
                  <h2 id="4-简化写法"><a href="#4-简化写法" class="headerlink" title="4. 简化写法"></a>4. 简化写法</h2>
                  <p>另外，文件写入还有一种简写方法，那就是使用<code>with as</code>语法。在<code>with</code>控制块结束时，文件会自动关闭，所以就不需要再调用<code>close()</code>方法了。这种保存方式可以简写如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">'explore.txt'</span>, <span class="string">'a'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> <span class="built_in">file</span>:</span><br><span class="line">    <span class="built_in">file</span>.<span class="built_in">write</span>(<span class="string">'\n'</span>.join([question, author, answer]))</span><br><span class="line">    <span class="built_in">file</span>.<span class="built_in">write</span>(<span class="string">'\n'</span> + <span class="string">'='</span> * <span class="number">50</span> + <span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果想保存时将原文清空，那么可以将第二个参数改写为<code>w</code>，代码如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">'explore.txt'</span>, <span class="string">'w'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> <span class="built_in">file</span>:</span><br><span class="line">    <span class="built_in">file</span>.<span class="built_in">write</span>(<span class="string">'\n'</span>.join([question, author, answer]))</span><br><span class="line">    <span class="built_in">file</span>.<span class="built_in">write</span>(<span class="string">'\n'</span> + <span class="string">'='</span> * <span class="number">50</span> + <span class="string">'\n'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>上面便是利用Python将结果保存为TXT文件的方法，这种方法简单易用，操作高效，是一种最基本的保存数据的方法。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 16:51:07" itemprop="dateCreated datePublished" datetime="2018-01-29T16:51:07+08:00">2018-01-29</time>
                </span>
                <span id="/5560.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.1.1-TXT文本存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>2.4k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>2 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5557.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5557.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5.1-文件存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>文件存储形式多种多样，比如可以保存成TXT纯文本形式，也可以保存为JSON格式、CSV格式等，本节就来了解一下文本文件的存储方式。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 16:43:41" itemprop="dateCreated datePublished" datetime="2018-01-29T16:43:41+08:00">2018-01-29</time>
                </span>
                <span id="/5557.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5.1-文件存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>65</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5554.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5554.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 5-数据存储</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>用解析器解析出数据之后，接下来就是存储数据了。保存的形式可以多种多样，最简单的形式是直接保存为文本文件，如TXT、JSON、CSV等。另外，还可以保存到数据库中，如关系型数据库MySQL，非关系型数据库MongoDB、Redis等。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-29 16:42:06" itemprop="dateCreated datePublished" datetime="2018-01-29T16:42:06+08:00">2018-01-29</time>
                </span>
                <span id="/5554.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 5-数据存储" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>116</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5551.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5551.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 4.3-使用pyquery</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在上一节中，我们介绍了Beautiful Soup的用法，它是一个非常强大的网页解析库，你是否觉得它的一些方法用起来有点不适应？有没有觉得它的CSS选择器的功能没有那么强大？</p>
                  <p>如果你对Web有所涉及，如果你比较喜欢用CSS选择器，如果你对jQuery有所了解，那么这里有一个更适合你的解析库——pyquery。</p>
                  <p>接下来，我们就来感受一下pyquery的强大之处。</p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>在开始之前，请确保已经正确安装好了pyquery。若没有安装，可以参考第1章的安装过程。</p>
                  <h2 id="2-初始化"><a href="#2-初始化" class="headerlink" title="2. 初始化"></a>2. 初始化</h2>
                  <p>像Beautiful Soup一样，初始化pyquery的时候，也需要传入HTML文本来初始化一个PyQuery对象。它的初始化方式有多种，比如直接传入字符串，传入URL，传入文件名，等等。下面我们来详细介绍一下。</p>
                  <h3 id="字符串初始化"><a href="#字符串初始化" class="headerlink" title="字符串初始化"></a>字符串初始化</h3>
                  <p>首先，我们用一个实例来感受一下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line">'''</span><br><span class="line">from pyquery import PyQuery as pq</span><br><span class="line">doc = pq(html)</span><br><span class="line">print(doc('li'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先引入PyQuery这个对象，取别名为<code>pq</code>。然后声明了一个长HTML字符串，并将其当作参数传递给<code>PyQuery</code>类，这样就成功完成了初始化。接下来，将初始化的对象传入CSS选择器。在这个实例中，我们传入<code>li</code>节点，这样就可以选择所有的<code>li</code>节点。</p>
                  <h3 id="URL初始化"><a href="#URL初始化" class="headerlink" title="URL初始化"></a>URL初始化</h3>
                  <p>初始化的参数不仅可以以字符串的形式传递，还可以传入网页的URL，此时只需要指定参数为<code>url</code>即可：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery as pq</span><br><span class="line">doc = pq(<span class="name">url=</span>'http<span class="symbol">://cuiqingcai</span>.com')</span><br><span class="line">print(<span class="name">doc</span>('title'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">title</span>&gt;</span>静觅丨崔庆才的个人博客<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样的话，<code>PyQuery</code>对象会首先请求这个URL，然后用得到的HTML内容完成初始化，这其实就相当于用网页的源代码以字符串的形式传递给<code>PyQuery</code>类来初始化。</p>
                  <p>它与下面的功能是相同的：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery as pq</span><br><span class="line">import requests</span><br><span class="line">doc = pq(<span class="name">requests</span>.get('http<span class="symbol">://cuiqingcai</span>.com').text)</span><br><span class="line">print(<span class="name">doc</span>('title'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="文件初始化"><a href="#文件初始化" class="headerlink" title="文件初始化"></a>文件初始化</h3>
                  <p>当然，除了传递URL，还可以传递本地的文件名，此时将参数指定为<code>filename</code>即可：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">filename</span>=<span class="string">'demo.html'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">doc</span>(<span class="string">'li'</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，这里需要有一个本地HTML文件demo.html，其内容是待解析的HTML字符串。这样它会首先读取本地的文件内容，然后用文件内容以字符串的形式传递给<code>PyQuery</code>类来初始化。</p>
                  <p>以上3种初始化方式均可，当然最常用的初始化方式还是以字符串形式传递。</p>
                  <h2 id="3-基本CSS选择器"><a href="#3-基本CSS选择器" class="headerlink" title="3. 基本CSS选择器"></a>3. 基本CSS选择器</h2>
                  <p>首先，用一个实例来感受pyquery的CSS选择器的用法：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;div id=<span class="string">"container"</span>&gt;</span><br><span class="line">    &lt;ul <span class="keyword">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;first item&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;second item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="keyword">class</span>=<span class="string">"bold"</span>&gt;third item&lt;/span&gt;&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;fourth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;fifth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">     &lt;/ul&gt;</span><br><span class="line"> &lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="keyword">print</span>(doc('#container .<span class="keyword">list</span> <span class="keyword">li</span>'))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">type</span>(doc('#container .<span class="keyword">list</span> <span class="keyword">li</span>')))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们初始化<code>PyQuery</code>对象之后，传入了一个CSS选择器<code>#container .list li</code>，它的意思是先选取<code>id</code>为<code>container</code>的节点，然后再选取其内部的<code>class</code>为<code>list</code>的节点内部的所有<code>li</code>节点。然后，打印输出。可以看到，我们成功获取到了符合条件的节点。</p>
                  <p>最后，将它的类型打印输出。可以看到，它的类型依然是<code>PyQuery</code>类型。</p>
                  <h2 id="4-查找节点"><a href="#4-查找节点" class="headerlink" title="4. 查找节点"></a>4. 查找节点</h2>
                  <p>下面我们介绍一些常用的查询函数，这些函数和jQuery中函数的用法完全相同。</p>
                  <h3 id="子节点"><a href="#子节点" class="headerlink" title="子节点"></a>子节点</h3>
                  <p>查找子节点时，需要用到<code>find()</code>方法，此时传入的参数是CSS选择器。这里还是以前面的HTML为例：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">html</span>)</span></span><br><span class="line"><span class="variable">items</span> = <span class="function"><span class="title">doc</span>(<span class="string">'.list'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">items</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">items</span>)</span></span><br><span class="line"><span class="variable">lis</span> = <span class="variable">items.find</span>(<span class="string">'li'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">lis</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">lis</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">ul</span> <span class="attr">class</span>=<span class="string">"list"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先，我们选取<code>class</code>为<code>list</code>的节点，然后调用了<code>find()</code>方法，传入CSS选择器，选取其内部的<code>li</code>节点，最后打印输出。可以发现，<code>find()</code>方法会将符合条件的所有节点选择出来，结果的类型是<code>PyQuery</code>类型。</p>
                  <p>其实<code>find()</code>的查找范围是节点的所有子孙节点，而如果我们只想查找子节点，那么可以用<code>children()</code>方法：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">lis</span> = items.children()</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">type</span>(<span class="keyword">lis</span>))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">lis</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果要筛选所有子节点中符合条件的节点，比如想筛选出子节点中<code>class</code>为<code>active</code>的节点，可以向<code>children()</code>方法传入CSS选择器<code>.active</code>：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">lis = items.children(<span class="string">'.active'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(lis)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，输出结果已经做了筛选，留下了<code>class</code>为<code>active</code>的节点。</p>
                  <h3 id="父节点"><a href="#父节点" class="headerlink" title="父节点"></a>父节点</h3>
                  <p>我们可以用<code>parent()</code>方法来获取某个节点的父节点，示例如下：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"wrap"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">id</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">        &lt;ul <span class="built_in">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0"</span>&gt;<span class="keyword">first</span> <span class="built_in">item</span>&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="built_in">class</span>=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="built_in">item</span>&lt;/span&gt;&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">         &lt;/ul&gt;</span><br><span class="line">     &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line"> &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">'''</span><br><span class="line"><span class="keyword">from</span> pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line">items = doc('.<span class="built_in">list</span>')</span><br><span class="line">container = items.parent()</span><br><span class="line">print(type(container))</span><br><span class="line">print(container)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"container"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span> <span class="attr">class</span>=<span class="string">"list"</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先用<code>.list</code>选取<code>class</code>为<code>list</code>的节点，然后调用<code>parent()</code>方法得到其父节点，其类型依然是<code>PyQuery</code>类型。</p>
                  <p>这里的父节点是该节点的直接父节点，也就是说，它不会再去查找父节点的父节点，即祖先节点。</p>
                  <p>但是如果想获取某个祖先节点，该怎么办呢？这时可以用<code>parents()</code>方法：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">html</span>)</span></span><br><span class="line"><span class="variable">items</span> = <span class="function"><span class="title">doc</span>(<span class="string">'.list'</span>)</span></span><br><span class="line"><span class="variable">parents</span> = <span class="variable">items.parents</span>()</span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">type</span>(<span class="variable">parents</span>))</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">parents</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">class</span>=<span class="string">"wrap"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"container"</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">ul</span> <span class="attr">class</span>=<span class="string">"list"</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"container"</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">ul</span> <span class="attr">class</span>=<span class="string">"list"</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，输出结果有两个：一个是<code>class</code>为<code>wrap</code>的节点，一个是<code>id</code>为<code>container</code>的节点。也就是说，<code>parents()</code>方法会返回所有的祖先节点。</p>
                  <p>如果想要筛选某个祖先节点的话，可以向<code>parents()</code>方法传入CSS选择器，这样就会返回祖先节点中符合CSS选择器的节点：</p>
                  <figure class="highlight mel">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">parent</span> = items.parents(<span class="string">'.wrap'</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">parent</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">div</span> <span class="attr">class</span>=<span class="string">"wrap"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"container"</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">ul</span> <span class="attr">class</span>=<span class="string">"list"</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">             <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，输出结果少了一个节点，只保留了<code>class</code>为<code>wrap</code>的节点。</p>
                  <h3 id="兄弟节点"><a href="#兄弟节点" class="headerlink" title="兄弟节点"></a>兄弟节点</h3>
                  <p>前面我们说明了子节点和父节点的用法，还有一种节点，那就是兄弟节点。如果要获取兄弟节点，可以使用<code>siblings()</code>方法。这里还是以上面的HTML代码为例：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">html</span>)</span></span><br><span class="line"><span class="variable">li</span> = <span class="function"><span class="title">doc</span>(<span class="string">'.list .item-0.active'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">li.siblings</span>())</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先选择<code>class</code>为<code>list</code>的节点内部<code>class</code>为<code>item-0</code>和<code>active</code>的节点，也就是第三个<code>li</code>节点。那么，很明显，它的兄弟节点有4个，那就是第一、二、四、五个<code>li</code>节点。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这正是我们刚才所说的4个兄弟节点。</p>
                  <p>如果要筛选某个兄弟节点，我们依然可以向<code>siblings</code>方法传入CSS选择器，这样就会从所有兄弟节点中挑选出符合条件的节点了：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">html</span>)</span></span><br><span class="line"><span class="variable">li</span> = <span class="function"><span class="title">doc</span>(<span class="string">'.list .item-0.active'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">li.siblings</span>(<span class="string">'.active'</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们筛选了<code>class</code>为<code>active</code>的节点，通过刚才的结果可以观察到，<code>class</code>为<code>active</code>的兄弟节点只有第四个<code>li</code>节点，所以结果应该是一个。</p>
                  <p>我们再看一下运行结果：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="5-遍历"><a href="#5-遍历" class="headerlink" title="5. 遍历"></a>5. 遍历</h2>
                  <p>刚才可以观察到，pyquery的选择结果可能是多个节点，也可能是单个节点，类型都是<code>PyQuery</code>类型，并没有返回像Beautiful Soup那样的列表。</p>
                  <p>对于单个节点来说，可以直接打印输出，也可以直接转成字符串：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">html</span>)</span></span><br><span class="line"><span class="variable">li</span> = <span class="function"><span class="title">doc</span>(<span class="string">'.item-0.active'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">li</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="title">str</span>(<span class="variable">li</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>对于多个节点的结果，我们就需要遍历来获取了。例如，这里把每一个<code>li</code>节点进行遍历，需要调用<code>items()</code>方法：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="keyword">lis</span> = doc('<span class="keyword">li</span>').items()</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">type</span>(<span class="keyword">lis</span>))</span><br><span class="line"><span class="keyword">for</span> <span class="keyword">li</span> <span class="keyword">in</span> <span class="keyword">lis</span>:</span><br><span class="line">    <span class="keyword">print</span>(<span class="keyword">li</span>, <span class="keyword">type</span>(<span class="keyword">li</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">generator</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，调用<code>items()</code>方法后，会得到一个生成器，遍历一下，就可以逐个得到<code>li</code>节点对象了，它的类型也是<code>PyQuery</code>类型。每个<code>li</code>节点还可以调用前面所说的方法进行选择，比如继续查询子节点，寻找某个祖先节点等，非常灵活。</p>
                  <h2 id="6-获取信息"><a href="#6-获取信息" class="headerlink" title="6. 获取信息"></a>6. 获取信息</h2>
                  <p>提取到节点之后，我们的最终目的当然是提取节点所包含的信息了。比较重要的信息有两类，一是获取属性，二是获取文本，下面分别进行说明。</p>
                  <h3 id="获取属性"><a href="#获取属性" class="headerlink" title="获取属性"></a>获取属性</h3>
                  <p>提取到某个<code>PyQuery</code>类型的节点后，就可以调用<code>attr()</code>方法来获取属性：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"wrap"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">id</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">        &lt;ul <span class="built_in">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0"</span>&gt;<span class="keyword">first</span> <span class="built_in">item</span>&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="built_in">class</span>=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="built_in">item</span>&lt;/span&gt;&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">         &lt;/ul&gt;</span><br><span class="line">     &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line"> &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">'''</span><br><span class="line"><span class="keyword">from</span> pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line">a = doc('.<span class="built_in">item</span><span class="number">-0.</span>active a')</span><br><span class="line">print(a, type(a))</span><br><span class="line">print(a.attr('href'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span> <span class="tag">&lt;<span class="name">class</span> '<span class="attr">pyquery.pyquery.PyQuery</span>'&gt;</span></span><br><span class="line">link3.html</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先选中<code>class</code>为<code>item-0</code>和<code>active</code>的<code>li</code>节点内的<code>a</code>节点，它的类型是<code>PyQuery</code>类型。</p>
                  <p>然后调用<code>attr()</code>方法。在这个方法中传入属性的名称，就可以得到这个属性值了。</p>
                  <p>此外，也可以通过调用<code>attr</code>属性来获取属性，用法如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">print</span><span class="params">(a.attr.href)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">link3</span><span class="selector-class">.html</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这两种方法的结果完全一样。</p>
                  <p>如果选中的是多个元素，然后调用<code>attr()</code>方法，会出现怎样的结果呢？我们用实例来测试一下：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">a = doc('a')</span><br><span class="line">print(<span class="name">a</span>, type(<span class="name">a</span>))</span><br><span class="line">print(<span class="name">a</span>.attr('href'))</span><br><span class="line">print(<span class="name">a</span>.attr.href)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="keyword">a</span> href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link3.html"</span>&gt;&lt;span class=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="keyword">item</span>&lt;/span&gt;&lt;/<span class="keyword">a</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;&lt;<span class="keyword">a</span> href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt; &lt;class <span class="string">'pyquery.pyquery.PyQuery'</span>&gt;</span><br><span class="line">link2.html</span><br><span class="line">link2.html</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>照理来说，我们选中的<code>a</code>节点应该有4个，而且打印结果也应该是4个，但是当我们调用<code>attr()</code>方法时，返回结果却只是第一个。这是因为，当返回结果包含多个节点时，调用<code>attr()</code>方法，只会得到第一个节点的属性。</p>
                  <p>那么，遇到这种情况时，如果想获取所有的<code>a</code>节点的属性，就要用到前面所说的遍历了：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">from</span> pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="keyword">a</span> = doc(<span class="string">'a'</span>)</span><br><span class="line"><span class="keyword">for</span> <span class="keyword">item</span> <span class="keyword">in</span> <span class="keyword">a</span>.<span class="keyword">items</span>():</span><br><span class="line">    print(<span class="keyword">item</span>.attr(<span class="string">'href'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此时的运行结果如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">link2</span><span class="selector-class">.html</span></span><br><span class="line"><span class="selector-tag">link3</span><span class="selector-class">.html</span></span><br><span class="line"><span class="selector-tag">link4</span><span class="selector-class">.html</span></span><br><span class="line"><span class="selector-tag">link5</span><span class="selector-class">.html</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因此，在进行属性获取时，可以观察返回节点是一个还是多个，如果是多个，则需要遍历才能依次获取每个节点的属性。</p>
                  <h3 id="获取文本"><a href="#获取文本" class="headerlink" title="获取文本"></a>获取文本</h3>
                  <p>获取节点之后的另一个主要操作就是获取其内部的文本了，此时可以调用<code>text()</code>方法来实现：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;<span class="keyword">div</span> <span class="built_in">class</span>=<span class="string">"wrap"</span>&gt;</span><br><span class="line">    &lt;<span class="keyword">div</span> <span class="built_in">id</span>=<span class="string">"container"</span>&gt;</span><br><span class="line">        &lt;ul <span class="built_in">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0"</span>&gt;<span class="keyword">first</span> <span class="built_in">item</span>&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="built_in">class</span>=<span class="string">"bold"</span>&gt;<span class="keyword">third</span> <span class="built_in">item</span>&lt;/span&gt;&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;<span class="keyword">fourth</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">             &lt;li <span class="built_in">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;<span class="keyword">fifth</span> <span class="built_in">item</span>&lt;/a&gt;&lt;/li&gt;</span><br><span class="line">         &lt;/ul&gt;</span><br><span class="line">     &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line"> &lt;/<span class="keyword">div</span>&gt;</span><br><span class="line">'''</span><br><span class="line"><span class="keyword">from</span> pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line">a = doc('.<span class="built_in">item</span><span class="number">-0.</span>active a')</span><br><span class="line">print(a)</span><br><span class="line">print(a.<span class="built_in">text</span>())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="keyword">class</span>="<span class="symbol">bold</span>"&gt;<span class="symbol">third</span> <span class="symbol">item</span>&lt;/<span class="symbol">span</span>&gt;&lt;/<span class="symbol">a</span>&gt;</span><br><span class="line"><span class="symbol">third</span> <span class="symbol">item</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先选中一个<code>a</code>节点，然后调用<code>text()</code>方法，就可以获取其内部的文本信息。此时它会忽略掉节点内部包含的所有HTML，只返回纯文字内容。</p>
                  <p>但如果想要获取这个节点内部的HTML文本，就要用<code>html()</code>方法了：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">from</span> <span class="variable">pyquery</span> <span class="variable">import</span> <span class="variable">PyQuery</span> <span class="variable">as</span> <span class="variable">pq</span></span><br><span class="line"><span class="variable">doc</span> = <span class="function"><span class="title">pq</span>(<span class="variable">html</span>)</span></span><br><span class="line"><span class="variable">li</span> = <span class="function"><span class="title">doc</span>(<span class="string">'.item-0.active'</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">li</span>)</span></span><br><span class="line"><span class="function"><span class="title">print</span>(<span class="variable">li.html</span>())</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们选中了第三个<code>li</code>节点，然后调用了<code>html()</code>方法，它返回的结果应该是<code>li</code>节点内的所有HTML文本。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="keyword">class</span>="<span class="symbol">bold</span>"&gt;<span class="symbol">third</span> <span class="symbol">item</span>&lt;/<span class="symbol">span</span>&gt;&lt;/<span class="symbol">a</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里同样有一个问题，如果我们选中的结果是多个节点，<code>text()</code>或<code>html()</code>会返回什么内容？我们用实例来看一下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;div <span class="keyword">class</span>=<span class="string">"wrap"</span>&gt;</span><br><span class="line">    &lt;div id=<span class="string">"container"</span>&gt;</span><br><span class="line">        &lt;ul <span class="keyword">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;second item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="keyword">class</span>=<span class="string">"bold"</span>&gt;third item&lt;/span&gt;&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;fourth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;fifth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;/ul&gt;</span><br><span class="line">     &lt;/div&gt;</span><br><span class="line"> &lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="keyword">li</span> = doc('<span class="keyword">li</span>')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>.html())</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>.text())</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">type</span>(<span class="keyword">li</span>.text())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="keyword">a</span> href=<span class="string">"link2.html"</span>&gt;<span class="keyword">second</span> <span class="keyword">item</span>&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line"><span class="keyword">second</span> <span class="keyword">item</span> <span class="keyword">third</span> <span class="keyword">item</span> <span class="keyword">fourth</span> <span class="keyword">item</span> <span class="keyword">fifth</span> <span class="keyword">item</span></span><br><span class="line">&lt;class <span class="string">'str'</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>结果可能比较出乎意料，<code>html()</code>方法返回的是第一个<code>li</code>节点的内部HTML文本，而<code>text()</code>则返回了所有的<code>li</code>节点内部的纯文本，中间用一个空格分割开，即返回结果是一个字符串。</p>
                  <p>所以这个地方值得注意，如果得到的结果是多个节点，并且想要获取每个节点的内部HTML文本，则需要遍历每个节点。而<code>text()</code>方法不需要遍历就可以获取，它将所有节点取文本之后合并成一个字符串。</p>
                  <h2 id="7-节点操作"><a href="#7-节点操作" class="headerlink" title="7. 节点操作"></a>7. 节点操作</h2>
                  <p>pyquery提供了一系列方法来对节点进行动态修改，比如为某个节点添加一个<code>class</code>，移除某个节点等，这些操作有时候会为提取信息带来极大的便利。</p>
                  <p>由于节点操作的方法太多，下面举几个典型的例子来说明它的用法。</p>
                  <h3 id="addClass和removeClass"><a href="#addClass和removeClass" class="headerlink" title="addClass和removeClass"></a><code>addClass</code>和<code>removeClass</code></h3>
                  <p>我们先用实例来感受一下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;div <span class="keyword">class</span>=<span class="string">"wrap"</span>&gt;</span><br><span class="line">    &lt;div id=<span class="string">"container"</span>&gt;</span><br><span class="line">        &lt;ul <span class="keyword">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;first item&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;second item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="keyword">class</span>=<span class="string">"bold"</span>&gt;third item&lt;/span&gt;&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;fourth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;fifth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;/ul&gt;</span><br><span class="line">     &lt;/div&gt;</span><br><span class="line"> &lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="keyword">li</span> = doc('.item-0.active')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br><span class="line"><span class="keyword">li</span>.removeClass('active')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br><span class="line"><span class="keyword">li</span>.addClass('active')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先选中了第三个<code>li</code>节点，然后调用<code>removeClass()</code>方法，将<code>li</code>节点的<code>active</code>这个<code>class</code>移除，后来又调用<code>addClass()</code>方法，将<code>class</code>添加回来。每执行一次操作，就打印输出当前<code>li</code>节点的内容。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，一共输出了3次。第二次输出时，<code>li</code>节点的<code>active</code>这个<code>class</code>被移除了，第三次<code>class</code>又添加回来了。</p>
                  <p>所以说，<code>addClass()</code>和<code>removeClass()</code>这些方法可以动态改变节点的<code>class</code>属性。</p>
                  <h3 id="attr、text和html"><a href="#attr、text和html" class="headerlink" title="attr、text和html"></a><code>attr</code>、<code>text</code>和<code>html</code></h3>
                  <p>当然，除了操作<code>class</code>这个属性外，也可以用<code>attr()</code>方法对属性进行操作。此外，还可以用<code>text()</code>和<code>html()</code>方法来改变节点内部的内容。示例如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = <span class="string">'''</span></span><br><span class="line"><span class="string">&lt;ul class="list"&gt;</span></span><br><span class="line"><span class="string">     &lt;li class="item-0 active"&gt;&lt;a href="link3.html"&gt;&lt;span class="bold"&gt;third item&lt;/span&gt;&lt;/a&gt;&lt;/li&gt;</span></span><br><span class="line"><span class="string">&lt;/ul&gt;</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"><span class="keyword">from</span> pyquery <span class="keyword">import</span> PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line">li = doc(<span class="string">'.item-0.active'</span>)</span><br><span class="line">print(li)</span><br><span class="line">li.attr(<span class="string">'name'</span>, <span class="string">'link'</span>)</span><br><span class="line">print(li)</span><br><span class="line">li.text(<span class="string">'changed item'</span>)</span><br><span class="line">print(li)</span><br><span class="line">li.html(<span class="string">'&lt;span&gt;changed item&lt;/span&gt;'</span>)</span><br><span class="line">print(li)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先选中<code>li</code>节点，然后调用<code>attr()</code>方法来修改属性，其中该方法的第一个参数为属性名，第二个参数为属性值。接着，调用<code>text()</code>和<code>html()</code>方法来改变节点内部的内容。三次操作后，分别打印输出当前的<code>li</code>节点。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span> <span class="attr">name</span>=<span class="string">"link"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span> <span class="attr">class</span>=<span class="string">"bold"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span> <span class="attr">name</span>=<span class="string">"link"</span>&gt;</span>changed item<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0 active"</span> <span class="attr">name</span>=<span class="string">"link"</span>&gt;</span><span class="tag">&lt;<span class="name">span</span>&gt;</span>changed item<span class="tag">&lt;/<span class="name">span</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，调用<code>attr()</code>方法后，<code>li</code>节点多了一个原本不存在的属性<code>name</code>，其值为<code>link</code>。接着调用<code>text()</code>方法，传入文本之后，<code>li</code>节点内部的文本全被改为传入的字符串文本了。最后，调用<code>html()</code>方法传入HTML文本后，<code>li</code>节点内部又变为传入的HTML文本了。</p>
                  <p>所以说，如果<code>attr()</code>方法只传入第一个参数的属性名，则是获取这个属性值；如果传入第二个参数，可以用来修改属性值。<code>text()</code>和<code>html()</code>方法如果不传参数，则是获取节点内纯文本和HTML文本；如果传入参数，则进行赋值。</p>
                  <h3 id="remove"><a href="#remove" class="headerlink" title="remove()"></a><code>remove()</code></h3>
                  <p>顾名思义，<code>remove()</code>方法就是移除，它有时会为信息的提取带来非常大的便利。下面有一段HTML文本：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = <span class="string">'''</span></span><br><span class="line"><span class="string">&lt;div class="wrap"&gt;</span></span><br><span class="line"><span class="string">    Hello, World</span></span><br><span class="line"><span class="string">    &lt;p&gt;This is a paragraph.&lt;/p&gt;</span></span><br><span class="line"><span class="string"> &lt;/div&gt;</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"><span class="keyword">from</span> pyquery <span class="keyword">import</span> PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line">wrap = doc(<span class="string">'.wrap'</span>)</span><br><span class="line">print(wrap.text())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在想提取<code>Hello, World</code>这个字符串，而不要<code>p</code>节点内部的字符串，需要怎样操作呢？</p>
                  <p>这里直接先尝试提取<code>class</code>为<code>wrap</code>的节点的内容，看看是不是我们想要的。运行结果如下：</p>
                  <figure class="highlight applescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Hello, World This <span class="keyword">is</span> a <span class="built_in">paragraph</span>.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个结果还包含了内部的<code>p</code>节点的内容，也就是说<code>text()</code>把所有的纯文本全提取出来了。如果我们想去掉<code>p</code>节点内部的文本，可以选择再把<code>p</code>节点内的文本提取一遍，然后从整个结果中移除这个子串，但这个做法明显比较烦琐。</p>
                  <p>这时<code>remove()</code>方法就可以派上用场了，我们可以接着这么做:</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">wrap.<span class="builtin-name">find</span>(<span class="string">'p'</span>).<span class="builtin-name">remove</span>()</span><br><span class="line"><span class="builtin-name">print</span>(wrap.text())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>首先选中<code>p</code>节点，然后调用了<code>remove()</code>方法将其移除，然后这时<code>wrap</code>内部就只剩下<code>Hello, World</code>这句话了，然后再利用<code>text()</code>方法提取即可。</p>
                  <p>另外，其实还有很多节点操作的方法，比如<code>append()</code>、<code>empty()</code>和<code>prepend()</code>等方法，它们和jQuery的用法完全一致，详细的用法可以参考官方文档：<a href="http://pyquery.readthedocs.io/en/latest/api.html" target="_blank" rel="noopener">http://pyquery.readthedocs.io/en/latest/api.html</a>。</p>
                  <h2 id="8-伪类选择器"><a href="#8-伪类选择器" class="headerlink" title="8. 伪类选择器"></a>8. 伪类选择器</h2>
                  <p>CSS选择器之所以强大，还有一个很重要的原因，那就是它支持多种多样的伪类选择器，例如选择第一个节点、最后一个节点、奇偶数节点、包含某一文本的节点等。示例如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''</span><br><span class="line">&lt;div <span class="keyword">class</span>=<span class="string">"wrap"</span>&gt;</span><br><span class="line">    &lt;div id=<span class="string">"container"</span>&gt;</span><br><span class="line">        &lt;ul <span class="keyword">class</span>=<span class="string">"list"</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;first item&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;second item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0 active"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;&lt;span <span class="keyword">class</span>=<span class="string">"bold"</span>&gt;third item&lt;/span&gt;&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1 active"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;fourth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">             &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;fifth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;/ul&gt;</span><br><span class="line">     &lt;/div&gt;</span><br><span class="line"> &lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from pyquery import PyQuery <span class="keyword">as</span> pq</span><br><span class="line">doc = pq(html)</span><br><span class="line"><span class="keyword">li</span> = doc('<span class="keyword">li</span>:first-child')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br><span class="line"><span class="keyword">li</span> = doc('<span class="keyword">li</span>:last-child')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br><span class="line"><span class="keyword">li</span> = doc('<span class="keyword">li</span>:nth-child(2)')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br><span class="line"><span class="keyword">li</span> = doc('<span class="keyword">li</span>:gt(2)')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)</span><br><span class="line"><span class="keyword">li</span> = doc('<span class="keyword">li</span>:nth-child(2n)')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>) `<span class="keyword">li</span> \= doc('<span class="keyword">li</span>:contains(second)')</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">li</span>)`</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用了CSS3的伪类选择器，依次选择了第一个<code>li</code>节点、最后一个<code>li</code>节点、第二个<code>li</code>节点、第三个<code>li</code>之后的<code>li</code>节点、偶数位置的<code>li</code>节点、包含<code>second</code>文本的<code>li</code>节点。</p>
                  <p>关于CSS选择器的更多用法，可以参考<a href="http://www.w3school.com.cn/css/index.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/css/index.asp</a>。</p>
                  <p>到此为止，pyquery的常用用法就介绍完了。如果想查看更多的内容，可以参考pyquery的官方文档：<a href="http://pyquery.readthedocs.io/" target="_blank" rel="noopener">http://pyquery.readthedocs.io</a>。我们相信有了它，解析网页不再是难事。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-28 16:31:40" itemprop="dateCreated datePublished" datetime="2018-01-28T16:31:40+08:00">2018-01-28</time>
                </span>
                <span id="/5551.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 4.3-使用pyquery" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>20k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>18 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5548.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5548.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 4.2-使用Beautiful Soup</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>前面介绍了正则表达式的相关用法，但是一旦正则表达式写的有问题，得到的可能就不是我们想要的结果了。而且对于一个网页来说，都有一定的特殊结构和层级关系，而且很多节点都有<code>id</code>或<code>class</code>来作区分，所以借助它们的结构和属性来提取不也可以吗？</p>
                  <p>这一节中，我们就来介绍一个强大的解析工具Beautiful Soup，它借助网页的结构和属性等特性来解析网页。有了它，我们不用再去写一些复杂的正则表达式，只需要简单的几条语句，就可以完成网页中某个元素的提取。</p>
                  <p>废话不多说，接下来就来感受一下Beautiful Soup的强大之处吧。</p>
                  <h2 id="1-简介"><a href="#1-简介" class="headerlink" title="1. 简介"></a>1. 简介</h2>
                  <p>简单来说，Beautiful Soup就是Python的一个HTML或XML的解析库，可以用它来方便地从网页中提取数据。官方解释如下：</p>
                  <blockquote>
                    <p>Beautiful Soup提供一些简单的、Python式的函数来处理导航、搜索、修改分析树等功能。它是一个工具箱，通过解析文档为用户提供需要抓取的数据，因为简单，所以不需要多少代码就可以写出一个完整的应用程序。</p>
                    <p>Beautiful Soup自动将输入文档转换为Unicode编码，输出文档转换为UTF-8编码。你不需要考虑编码方式，除非文档没有指定一个编码方式，这时你仅仅需要说明一下原始编码方式就可以了。</p>
                    <p>Beautiful Soup已成为和lxml、html6lib一样出色的Python解释器，为用户灵活地提供不同的解析策略或强劲的速度。</p>
                  </blockquote>
                  <p>所以说，利用它可以省去很多烦琐的提取工作，提高了解析效率。</p>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>在开始之前，请确保已经正确安装好了Beautiful Soup和lxml，如果没有安装，可以参考第1章的内容。</p>
                  <h2 id="3-解析器"><a href="#3-解析器" class="headerlink" title="3. 解析器"></a>3. 解析器</h2>
                  <p>Beautiful Soup在解析时实际上依赖解析器，它除了支持Python标准库中的HTML解析器外，还支持一些第三方解析器（比如lxml）。表4-3列出了Beautiful Soup支持的解析器。</p>
                  <p>表4-3 Beautiful Soup支持的解析器</p>
                  <p>解析器</p>
                  <p>使用方法</p>
                  <p>优势</p>
                  <p>劣势</p>
                  <p>Python标准库</p>
                  <p><code>BeautifulSoup(markup, &quot;html.parser&quot;)</code></p>
                  <p>Python的内置标准库、执行速度适中、文档容错能力强</p>
                  <p>Python 2.7.3及Python 3.2.2之前的版本文档容错能力差</p>
                  <p>lxml HTML解析器</p>
                  <p><code>BeautifulSoup(markup, &quot;lxml&quot;)</code></p>
                  <p>速度快、文档容错能力强</p>
                  <p>需要安装C语言库</p>
                  <p>lxml XML解析器</p>
                  <p><code>BeautifulSoup(markup, &quot;xml&quot;)</code></p>
                  <p>速度快、唯一支持XML的解析器</p>
                  <p>需要安装C语言库</p>
                  <p>html5lib</p>
                  <p><code>BeautifulSoup(markup, &quot;html5lib&quot;)</code></p>
                  <p>最好的容错性、以浏览器的方式解析文档、生成HTML5格式的文档</p>
                  <p>速度慢、不依赖外部扩展</p>
                  <p>通过以上对比可以看出，lxml解析器有解析HTML和XML的功能，而且速度快，容错能力强，所以推荐使用它。</p>
                  <p>如果使用lxml，那么在初始化Beautiful Soup时，可以把第二个参数改为<code>lxml</code>即可：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(<span class="string">'&lt;p&gt;Hello&lt;/p&gt;'</span>, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="built_in">print</span>(soup.p.<span class="built_in">string</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在后面，Beautiful Soup的用法实例也统一用这个解析器来演示。</p>
                  <h2 id="4-基本用法"><a href="#4-基本用法" class="headerlink" title="4. 基本用法"></a>4. 基本用法</h2>
                  <p>下面首先用实例来看看Beautiful Soup的基本用法：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = """</span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span><span class="tag">&lt;<span class="name">head</span>&gt;</span><span class="tag">&lt;<span class="name">title</span>&gt;</span>The Dormouse's story<span class="tag">&lt;/<span class="name">title</span>&gt;</span><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"title"</span> <span class="attr">name</span>=<span class="string">"dromouse"</span>&gt;</span><span class="tag">&lt;<span class="name">b</span>&gt;</span>The Dormouse's story<span class="tag">&lt;/<span class="name">b</span>&gt;</span><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span>Once upon a time there were three little sisters; and their names were</span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span><span class="comment">&lt;!-- Elsie --&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span>,</span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/lacie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link2"</span>&gt;</span>Lacie<span class="tag">&lt;/<span class="name">a</span>&gt;</span> and</span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/tillie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link3"</span>&gt;</span>Tillie<span class="tag">&lt;/<span class="name">a</span>&gt;</span>;</span><br><span class="line">and they lived at the bottom of a well.<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span>...<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">"""</span><br><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, 'lxml')</span><br><span class="line">print(soup.prettify())</span><br><span class="line">print(soup.title.string)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">title</span>&gt;</span></span><br><span class="line">   The Dormouse's story</span><br><span class="line">  <span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"title"</span> <span class="attr">name</span>=<span class="string">"dromouse"</span>&gt;</span></span><br><span class="line">   <span class="tag">&lt;<span class="name">b</span>&gt;</span></span><br><span class="line">    The Dormouse's story</span><br><span class="line">   <span class="tag">&lt;/<span class="name">b</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line">   Once upon a time there were three little sisters; and their names were</span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line">    <span class="comment">&lt;!-- Elsie --&gt;</span></span><br><span class="line">   <span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   ,</span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/lacie"</span> <span class="attr">id</span>=<span class="string">"link2"</span>&gt;</span></span><br><span class="line">    Lacie</span><br><span class="line">   <span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   and</span><br><span class="line">   <span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/tillie"</span> <span class="attr">id</span>=<span class="string">"link3"</span>&gt;</span></span><br><span class="line">    Tillie</span><br><span class="line">   <span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">   ;</span><br><span class="line">and they lived at the bottom of a well.</span><br><span class="line">  <span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">  <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line">   ...</span><br><span class="line">  <span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br><span class="line">The Dormouse's story</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先声明变量<code>html</code>，它是一个HTML字符串。但是需要注意的是，它并不是一个完整的HTML字符串，因为<code>body</code>和<code>html</code>节点都没有闭合。接着，我们将它当作第一个参数传给<code>BeautifulSoup</code>对象，该对象的第二个参数为解析器的类型（这里使用<code>lxml</code>），此时就完成了<code>BeaufulSoup</code>对象的初始化。然后，将这个对象赋值给<code>soup</code>变量。</p>
                  <p>接下来，就可以调用<code>soup</code>的各个方法和属性解析这串HTML代码了。</p>
                  <p>首先，调用<code>prettify()</code>方法。这个方法可以把要解析的字符串以标准的缩进格式输出。这里需要注意的是，输出结果里面包含<code>body</code>和<code>html</code>节点，也就是说对于不标准的HTML字符串<code>BeautifulSoup</code>，可以自动更正格式。这一步不是由<code>prettify()</code>方法做的，而是在初始化<code>BeautifulSoup</code>时就完成了。</p>
                  <p>然后调用<code>soup.title.string</code>，这实际上是输出HTML中<code>title</code>节点的文本内容。所以，<code>soup.title</code>可以选出HTML中的<code>title</code>节点，再调用<code>string</code>属性就可以得到里面的文本了，所以我们可以通过简单调用几个属性完成文本提取，这是不是非常方便？</p>
                  <h2 id="5-节点选择器"><a href="#5-节点选择器" class="headerlink" title="5. 节点选择器"></a>5. 节点选择器</h2>
                  <p>直接调用节点的名称就可以选择节点元素，再调用<code>string</code>属性就可以得到节点内的文本了，这种选择方式速度非常快。如果单个节点结构层次非常清晰，可以选用这种方式来解析。</p>
                  <h3 id="选择元素"><a href="#选择元素" class="headerlink" title="选择元素"></a>选择元素</h3>
                  <p>下面再用一个例子详细说明选择元素的方法：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = """</span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span><span class="tag">&lt;<span class="name">head</span>&gt;</span><span class="tag">&lt;<span class="name">title</span>&gt;</span>The Dormouse's story<span class="tag">&lt;/<span class="name">title</span>&gt;</span><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"title"</span> <span class="attr">name</span>=<span class="string">"dromouse"</span>&gt;</span><span class="tag">&lt;<span class="name">b</span>&gt;</span>The Dormouse's story<span class="tag">&lt;/<span class="name">b</span>&gt;</span><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span>Once upon a time there were three little sisters; and their names were</span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span><span class="comment">&lt;!-- Elsie --&gt;</span><span class="tag">&lt;/<span class="name">a</span>&gt;</span>,</span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/lacie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link2"</span>&gt;</span>Lacie<span class="tag">&lt;/<span class="name">a</span>&gt;</span> and</span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/tillie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link3"</span>&gt;</span>Tillie<span class="tag">&lt;/<span class="name">a</span>&gt;</span>;</span><br><span class="line">and they lived at the bottom of a well.<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span>...<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">"""</span><br><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, 'lxml')</span><br><span class="line">print(soup.title)</span><br><span class="line">print(type(soup.title))</span><br><span class="line">print(soup.title.string)</span><br><span class="line">print(soup.head)</span><br><span class="line">print(soup.p)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;title&gt;The Dormous<span class="string">e's story&lt;/title&gt;</span></span><br><span class="line"><span class="string">&lt;class '</span>bs4.element.Tag<span class="string">'&gt;</span></span><br><span class="line"><span class="string">The Dormouse'</span>s story</span><br><span class="line">&lt;head&gt;&lt;title&gt;The Dormous<span class="string">e's story&lt;/title&gt;&lt;/head&gt;</span></span><br><span class="line"><span class="string">&lt;p class="title" name="dromouse"&gt;&lt;b&gt;The Dormouse'</span>s story&lt;/b&gt;&lt;/p&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里依然选用刚才的HTML代码，首先打印输出<code>title</code>节点的选择结果，输出结果正是<code>title</code>节点加里面的文字内容。接下来，输出它的类型，是<code>bs4.element.Tag</code>类型，这是Beautiful Soup中一个重要的数据结构。经过选择器选择后，选择结果都是这种<code>Tag</code>类型。<code>Tag</code>具有一些属性，比如<code>string</code>属性，调用该属性，可以得到节点的文本内容，所以接下来的输出结果正是节点的文本内容。</p>
                  <p>接下来，我们又尝试选择了<code>head</code>节点，结果也是节点加其内部的所有内容。最后，选择了<code>p</code>节点。不过这次情况比较特殊，我们发现结果是第一个<code>p</code>节点的内容，后面的几个<code>p</code>节点并没有选到。也就是说，当有多个节点时，这种选择方式只会选择到第一个匹配的节点，其他的后面节点都会忽略。</p>
                  <h3 id="提取信息"><a href="#提取信息" class="headerlink" title="提取信息"></a>提取信息</h3>
                  <p>上面演示了调用<code>string</code>属性来获取文本的值，那么如何获取节点属性的值呢？如何获取节点名呢？下面我们来统一梳理一下信息的提取方式。</p>
                  <h4 id="1-获取名称"><a href="#1-获取名称" class="headerlink" title="(1)获取名称"></a>(1)获取名称</h4>
                  <p>可以利用<code>name</code>属性获取节点的名称。这里还是以上面的文本为例，选取<code>title</code>节点，然后调用<code>name</code>属性就可以得到节点名称：</p>
                  <figure class="highlight gauss">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">print</span>(soup.<span class="built_in">title</span>.name)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight dos">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">title</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h4 id="2-获取属性"><a href="#2-获取属性" class="headerlink" title="(2)获取属性"></a>(2)获取属性</h4>
                  <p>每个节点可能有多个属性，比如<code>id</code>和<code>class</code>等，选择这个节点元素后，可以调用<code>attrs</code>获取所有属性：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p.attrs)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p.attrs[<span class="string">'name'</span>])</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight prolog">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;<span class="string">'class'</span>: [<span class="string">'title'</span>], <span class="string">'name'</span>: <span class="string">'dromouse'</span>&#125;</span><br><span class="line">dromouse</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，<code>attrs</code>的返回结果是字典形式，它把选择的节点的所有属性和属性值组合成一个字典。接下来，如果要获取<code>name</code>属性，就相当于从字典中获取某个键值，只需要用中括号加属性名就可以了。比如，要获取<code>name</code>属性，就可以通过<code>attrs[&#39;name&#39;]</code>来得到。</p>
                  <p>其实这样有点烦琐，还有一种更简单的获取方式：可以不用写<code>attrs</code>，直接在节点元素后面加中括号，传入属性名就可以获取属性值了。样例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p[<span class="string">'name'</span>])</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p[<span class="string">'class'</span>])</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">dromouse</span></span><br><span class="line"><span class="selector-attr">[<span class="string">'title'</span>]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里需要注意的是，有的返回结果是字符串，有的返回结果是字符串组成的列表。比如，<code>name</code>属性的值是唯一的，返回的结果就是单个字符串。而对于<code>class</code>，一个节点元素可能有多个<code>class</code>，所以返回的是列表。在实际处理过程中，我们要注意判断类型。</p>
                  <h4 id="3-获取内容"><a href="#3-获取内容" class="headerlink" title="(3)获取内容"></a>(3)获取内容</h4>
                  <p>可以利用<code>string</code>属性获取节点元素包含的文本内容，比如要获取第一个<code>p</code>节点的文本：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p.string)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ada">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">The Dormouse<span class="symbol">'s</span> story</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再次注意一下，这里选择到的<code>p</code>节点是第一个<code>p</code>节点，获取的文本也是第一个<code>p</code>节点里面的文本。</p>
                  <h3 id="嵌套选择"><a href="#嵌套选择" class="headerlink" title="嵌套选择"></a>嵌套选择</h3>
                  <p>在上面的例子中，我们知道每一个返回结果都是<code>bs4.element.Tag</code>类型，它同样可以继续调用节点进行下一步的选择。比如，我们获取了<code>head</code>节点元素，我们可以继续调用<code>head</code>来选取其内部的<code>head</code>节点元素：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = <span class="string">"""</span></span><br><span class="line"><span class="string">&lt;html&gt;&lt;head&gt;&lt;title&gt;The Dormouse's story&lt;/title&gt;&lt;/head&gt;</span></span><br><span class="line"><span class="string">&lt;body&gt;</span></span><br><span class="line"><span class="string">"""</span></span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line">print(soup.head.title)</span><br><span class="line">print(type(soup.head.title))</span><br><span class="line">print(soup.head.title.string)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;title&gt;The Dormous<span class="string">e's story&lt;/title&gt;</span></span><br><span class="line"><span class="string">&lt;class '</span>bs4.element.Tag<span class="string">'&gt;</span></span><br><span class="line"><span class="string">The Dormouse'</span>s story</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一行结果是调用<code>head</code>之后再次调用<code>title</code>而选择的<code>title</code>节点元素。然后打印输出了它的类型，可以看到，它仍然是<code>bs4.element.Tag</code>类型。也就是说，我们在<code>Tag</code>类型的基础上再次选择得到的依然还是<code>Tag</code>类型，每次返回的结果都相同，所以这样就可以做嵌套选择了。</p>
                  <p>最后，输出它的<code>string</code>属性，也就是节点里的文本内容。</p>
                  <h3 id="关联选择"><a href="#关联选择" class="headerlink" title="关联选择"></a>关联选择</h3>
                  <p>在做选择的时候，有时候不能做到一步就选到想要的节点元素，需要先选中某一个节点元素，然后以它为基准再选择它的子节点、父节点、兄弟节点等，这里就来介绍如何选择这些节点元素。</p>
                  <h4 id="1-子节点和子孙节点"><a href="#1-子节点和子孙节点" class="headerlink" title="(1)子节点和子孙节点"></a>(1)子节点和子孙节点</h4>
                  <p>选取节点元素之后，如果想要获取它的直接子节点，可以调用<code>contents</code>属性，示例如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = """</span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">title</span>&gt;</span>The Dormouse's story<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line">            Once upon a time there were three little sisters; and their names were</span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/lacie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link2"</span>&gt;</span>Lacie<span class="tag">&lt;/<span class="name">a</span>&gt;</span> </span><br><span class="line">            and</span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/tillie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link3"</span>&gt;</span>Tillie<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">            and they lived at the bottom of a well.</span><br><span class="line">        <span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span>...<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">"""</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xquery">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="string">'\n            Once upon a time there were three little sisters; and their names were\n            '</span>, <span class="xml"><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span>, <span class="string">'\n'</span>, <span class="xml"><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/lacie"</span> <span class="attr">id</span>=<span class="string">"link2"</span>&gt;</span>Lacie<span class="tag">&lt;/<span class="name">a</span>&gt;</span>, ' \n            and\n            ', <span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/tillie"</span> <span class="attr">id</span>=<span class="string">"link3"</span>&gt;</span>Tillie<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span>, <span class="string">'\n            and they lived at the bottom of a well.\n        '</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，返回结果是列表形式。<code>p</code>节点里既包含文本，又包含节点，最后会将它们以列表形式统一返回。</p>
                  <p>需要注意的是，列表中的每个元素都是<code>p</code>节点的直接子节点。比如第一个<code>a</code>节点里面包含一层<code>span</code>节点，这相当于孙子节点了，但是返回结果并没有单独把<code>span</code>节点选出来。所以说，<code>contents</code>属性得到的结果是直接子节点的列表。</p>
                  <p>同样，我们可以调用<code>children</code>属性得到相应的结果：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(<span class="selector-tag">html</span>, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p.children)</span></span></span><br><span class="line"><span class="keyword">for</span> <span class="selector-tag">i</span>, child <span class="keyword">in</span> enumerate(soup<span class="selector-class">.p</span>.children):</span><br><span class="line">    print(<span class="selector-tag">i</span>, child)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;list_iterator object <span class="keyword">at</span> <span class="number">0x1064f7dd8</span>&gt;</span><br><span class="line"><span class="number">0</span> </span><br><span class="line">            Once upon <span class="keyword">a</span> <span class="built_in">time</span> there were <span class="literal">three</span> little sisters; <span class="keyword">and</span> their names were</span><br><span class="line"></span><br><span class="line"><span class="number">1</span> &lt;<span class="keyword">a</span> class=<span class="string">"sister"</span> href=<span class="string">"http://example.com/elsie"</span> id=<span class="string">"link1"</span>&gt;</span><br><span class="line">&lt;span&gt;Elsie&lt;/span&gt;</span><br><span class="line">&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line"><span class="number">2</span> </span><br><span class="line"></span><br><span class="line"><span class="number">3</span> &lt;<span class="keyword">a</span> class=<span class="string">"sister"</span> href=<span class="string">"http://example.com/lacie"</span> id=<span class="string">"link2"</span>&gt;Lacie&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line"><span class="number">4</span>  </span><br><span class="line">            <span class="keyword">and</span></span><br><span class="line"></span><br><span class="line"><span class="number">5</span> &lt;<span class="keyword">a</span> class=<span class="string">"sister"</span> href=<span class="string">"http://example.com/tillie"</span> id=<span class="string">"link3"</span>&gt;Tillie&lt;/<span class="keyword">a</span>&gt;</span><br><span class="line"><span class="number">6</span> </span><br><span class="line">            <span class="keyword">and</span> they lived <span class="keyword">at</span> <span class="keyword">the</span> bottom <span class="keyword">of</span> <span class="keyword">a</span> well.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>还是同样的HTML文本，这里调用了<code>children</code>属性来选择，返回结果是生成器类型。接下来，我们用<code>for</code>循环输出相应的内容。</p>
                  <p>如果要得到所有的子孙节点的话，可以调用<code>descendants</code>属性：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(<span class="selector-tag">html</span>, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(soup.p.descendants)</span></span></span><br><span class="line"><span class="keyword">for</span> <span class="selector-tag">i</span>, child <span class="keyword">in</span> enumerate(soup<span class="selector-class">.p</span>.descendants):</span><br><span class="line">    print(<span class="selector-tag">i</span>, child)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;generator object descendants at <span class="number">0x10650e678</span>&gt;</span><br><span class="line"><span class="number">0</span> </span><br><span class="line">            Once upon a time there were three little sisters; and their names were</span><br><span class="line"></span><br><span class="line"><span class="number">1</span> &lt;a class=<span class="string">"sister"</span> href=<span class="string">"http://example.com/elsie"</span> id=<span class="string">"link1"</span>&gt;</span><br><span class="line">&lt;span&gt;Elsie&lt;/span&gt;</span><br><span class="line">&lt;/a&gt;</span><br><span class="line"><span class="number">2</span> </span><br><span class="line"></span><br><span class="line"><span class="number">3</span> &lt;span&gt;Elsie&lt;/span&gt;</span><br><span class="line"><span class="number">4</span> Elsie</span><br><span class="line"><span class="number">5</span> </span><br><span class="line"></span><br><span class="line"><span class="number">6</span> </span><br><span class="line"></span><br><span class="line"><span class="number">7</span> &lt;a class=<span class="string">"sister"</span> href=<span class="string">"http://example.com/lacie"</span> id=<span class="string">"link2"</span>&gt;Lacie&lt;/a&gt;</span><br><span class="line"><span class="number">8</span> Lacie</span><br><span class="line"><span class="number">9</span>  </span><br><span class="line">            and</span><br><span class="line"></span><br><span class="line"><span class="number">10</span> &lt;a class=<span class="string">"sister"</span> href=<span class="string">"http://example.com/tillie"</span> id=<span class="string">"link3"</span>&gt;Tillie&lt;/a&gt;</span><br><span class="line"><span class="number">11</span> Tillie</span><br><span class="line"><span class="number">12</span> </span><br><span class="line">            and they lived at the bottom of a well.</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此时返回结果还是生成器。遍历输出一下可以看到，这次的输出结果就包含了<code>span</code>节点。<code>descendants</code>会递归查询所有子节点，得到所有的子孙节点。</p>
                  <h4 id="2-父节点和祖先节点"><a href="#2-父节点和祖先节点" class="headerlink" title="(2)父节点和祖先节点"></a>(2)父节点和祖先节点</h4>
                  <p>如果要获取某个节点元素的父节点，可以调用<code>parent</code>属性：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = """</span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">head</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">title</span>&gt;</span>The Dormouse's story<span class="tag">&lt;/<span class="name">title</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">head</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line">            Once upon a time there were three little sisters; and their names were</span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line">                <span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span>...<span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">"""</span><br><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, 'lxml')</span><br><span class="line">print(soup.a.parent)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line">            Once upon a time there were three little sisters; and their names were</span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们选择的是第一个<code>a</code>节点的父节点元素。很明显，它的父节点是<code>p</code>节点，输出结果便是<code>p</code>节点及其内部的内容。</p>
                  <p>需要注意的是，这里输出的仅仅是<code>a</code>节点的直接父节点，而没有再向外寻找父节点的祖先节点。如果想获取所有的祖先节点，可以调用<code>parents</code>属性：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = <span class="string">""</span><span class="comment">"</span></span><br><span class="line"><span class="symbol">&lt;html&gt;</span></span><br><span class="line">    <span class="symbol">&lt;body&gt;</span></span><br><span class="line">        &lt;<span class="keyword">p</span> class=<span class="string">"story"</span>&gt;</span><br><span class="line">            &lt;<span class="keyword">a</span> href=<span class="string">"http://example.com/elsie"</span> class=<span class="string">"sister"</span> id=<span class="string">"link1"</span>&gt;</span><br><span class="line">                <span class="symbol">&lt;span&gt;</span>Elsie&lt;/span&gt;</span><br><span class="line">            &lt;/<span class="keyword">a</span>&gt;</span><br><span class="line">        &lt;/<span class="keyword">p</span>&gt;</span><br><span class="line"><span class="string">""</span><span class="comment">"</span></span><br><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="keyword">print</span>(<span class="built_in">type</span>(soup.<span class="keyword">a</span>.parents))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">list</span>(enumerate(soup.<span class="keyword">a</span>.parents)))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">generator</span>'&gt;</span></span><br><span class="line">[(0, <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">p</span>&gt;</span>), (1, <span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span>), (2, <span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="tag">&lt;/<span class="name">html</span>&gt;</span>), (3, <span class="tag">&lt;<span class="name">html</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"story"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">a</span> <span class="attr">class</span>=<span class="string">"sister"</span> <span class="attr">href</span>=<span class="string">"http://example.com/elsie"</span> <span class="attr">id</span>=<span class="string">"link1"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">span</span>&gt;</span>Elsie<span class="tag">&lt;/<span class="name">span</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="tag">&lt;/<span class="name">html</span>&gt;</span>)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，返回结果是生成器类型。这里用列表输出了它的索引和内容，而列表中的元素就是<code>a</code>节点的祖先节点。</p>
                  <h4 id="3-兄弟节点"><a href="#3-兄弟节点" class="headerlink" title="(3)兄弟节点"></a>(3)兄弟节点</h4>
                  <p>上面说明了子节点和父节点的获取方式，如果要获取同级的节点（也就是兄弟节点），应该怎么办呢？示例如下：</p>
                  <figure class="highlight smalltalk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = <span class="comment">""</span><span class="comment">"</span></span><br><span class="line"><span class="comment">&lt;html&gt;</span></span><br><span class="line"><span class="comment">    &lt;body&gt;</span></span><br><span class="line"><span class="comment">        &lt;p class="</span>story<span class="comment">"&gt;</span></span><br><span class="line"><span class="comment">            Once upon a time there were three little sisters; and their names were</span></span><br><span class="line"><span class="comment">            &lt;a href="</span>http://example.com/elsie<span class="comment">" class="</span>sister<span class="comment">" id="</span>link1<span class="comment">"&gt;</span></span><br><span class="line"><span class="comment">                &lt;span&gt;Elsie&lt;/span&gt;</span></span><br><span class="line"><span class="comment">            &lt;/a&gt;</span></span><br><span class="line"><span class="comment">            Hello</span></span><br><span class="line"><span class="comment">            &lt;a href="</span>http://example.com/lacie<span class="comment">" class="</span>sister<span class="comment">" id="</span>link2<span class="comment">"&gt;Lacie&lt;/a&gt; </span></span><br><span class="line"><span class="comment">            and</span></span><br><span class="line"><span class="comment">            &lt;a href="</span>http://example.com/tillie<span class="comment">" class="</span>sister<span class="comment">" id="</span>link3<span class="comment">"&gt;Tillie&lt;/a&gt;</span></span><br><span class="line"><span class="comment">            and they lived at the bottom of a well.</span></span><br><span class="line"><span class="comment">        &lt;/p&gt;</span></span><br><span class="line"><span class="comment">"</span><span class="comment">""</span></span><br><span class="line">from bs4 import <span class="type">BeautifulSoup</span></span><br><span class="line">soup = <span class="type">BeautifulSoup</span>(html, <span class="string">'lxml'</span>)</span><br><span class="line">print(<span class="string">'Next Sibling'</span>, soup.a.next_sibling)</span><br><span class="line">print(<span class="string">'Prev Sibling'</span>, soup.a.previous_sibling)</span><br><span class="line">print(<span class="string">'Next Siblings'</span>, list(enumerate(soup.a.next_siblings)))</span><br><span class="line">print(<span class="string">'Prev Siblings'</span>, list(enumerate(soup.a.previous_siblings)))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Next Sibling </span><br><span class="line">            Hello</span><br><span class="line"></span><br><span class="line">Prev Sibling </span><br><span class="line">            Once upon a time there were three little sisters; <span class="keyword">and</span> their names were</span><br><span class="line"></span><br><span class="line">Next Siblings [(0, <span class="string">'\n            Hello\n            '</span>), (1, &lt;a <span class="attribute">class</span>=<span class="string">"sister"</span> <span class="attribute">href</span>=<span class="string">"http://example.com/lacie"</span> <span class="attribute">id</span>=<span class="string">"link2"</span>&gt;Lacie&lt;/a&gt;), (2, <span class="string">' \n            and\n            '</span>), (3, &lt;a <span class="attribute">class</span>=<span class="string">"sister"</span> <span class="attribute">href</span>=<span class="string">"http://example.com/tillie"</span> <span class="attribute">id</span>=<span class="string">"link3"</span>&gt;Tillie&lt;/a&gt;), (4, <span class="string">'\n            and they lived at the bottom of a well.\n        '</span>)]</span><br><span class="line">Prev Siblings [(0, <span class="string">'\n            Once upon a time there were three little sisters; and their names were\n            '</span>)]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里调用了4个属性，其中<code>next_sibling</code>和<code>previous_sibling</code>分别获取节点的下一个和上一个兄弟元素，<code>next_siblings</code>和<code>previous_siblings</code>则分别返回所有前面和后面的兄弟节点的生成器。</p>
                  <h4 id="4-提取信息"><a href="#4-提取信息" class="headerlink" title="(4)提取信息"></a>(4)提取信息</h4>
                  <p>前面讲解了关联元素节点的选择方法，如果想要获取它们的一些信息，比如文本、属性等，也用同样的方法，示例如下：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = <span class="string">""</span><span class="string">"</span></span><br><span class="line"><span class="string">&lt;html&gt;</span></span><br><span class="line"><span class="string">    &lt;body&gt;</span></span><br><span class="line"><span class="string">        &lt;p class="</span>story<span class="string">"&gt;</span></span><br><span class="line"><span class="string">            Once upon a time there were three little sisters; and their names were</span></span><br><span class="line"><span class="string">            &lt;a href="</span><span class="name">http</span>://example.com/elsie<span class="string">" class="</span>sister<span class="string">" id="</span>link1<span class="string">"&gt;Bob&lt;/a&gt;&lt;a href="</span><span class="name">http</span>://example.com/lacie<span class="string">" class="</span>sister<span class="string">" id="</span>link2<span class="string">"&gt;Lacie&lt;/a&gt; </span></span><br><span class="line"><span class="string">        &lt;/p&gt;</span></span><br><span class="line"><span class="string">"</span><span class="string">""</span></span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="built_in">print</span>(<span class="string">'Next Sibling:'</span>)</span><br><span class="line"><span class="built_in">print</span>(<span class="built_in">type</span>(soup.a.next_sibling))</span><br><span class="line"><span class="built_in">print</span>(soup.a.next_sibling)</span><br><span class="line"><span class="built_in">print</span>(soup.a.next_sibling.<span class="built_in">string</span>)</span><br><span class="line"><span class="built_in">print</span>(<span class="string">'Parent:'</span>)</span><br><span class="line"><span class="built_in">print</span>(<span class="built_in">type</span>(soup.a.parents))</span><br><span class="line"><span class="built_in">print</span>(list(soup.a.parents)[<span class="number">0</span>])</span><br><span class="line"><span class="built_in">print</span>(list(soup.a.parents)[<span class="number">0</span>].attrs[<span class="string">'class'</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scala">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="type">Next</span> <span class="type">Sibling</span>:</span><br><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">bs4</span>.<span class="title">element</span>.<span class="title">Tag</span>'<span class="title">&gt;</span></span></span><br><span class="line"><span class="class"><span class="title">&lt;a</span> <span class="title">class=</span>"<span class="title">sister</span>" <span class="title">href=</span>"<span class="title">http</span></span>:<span class="comment">//example.com/lacie" id="link2"&gt;Lacie&lt;/a&gt;</span></span><br><span class="line"><span class="type">Lacie</span></span><br><span class="line"><span class="type">Parent</span>:</span><br><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">generator</span>'<span class="title">&gt;</span></span></span><br><span class="line"><span class="class"><span class="title">&lt;p</span> <span class="title">class=</span>"<span class="title">story</span>"<span class="title">&gt;</span></span></span><br><span class="line"><span class="class">            <span class="title">Once</span> <span class="title">upon</span> <span class="title">a</span> <span class="title">time</span> <span class="title">there</span> <span class="title">were</span> <span class="title">three</span> <span class="title">little</span> <span class="title">sisters</span></span>; and their names were</span><br><span class="line">            &lt;a <span class="class"><span class="keyword">class</span></span>=<span class="string">"sister"</span> href=<span class="string">"http://example.com/elsie"</span> id=<span class="string">"link1"</span>&gt;<span class="type">Bob</span>&lt;/a&gt;&lt;a <span class="class"><span class="keyword">class</span></span>=<span class="string">"sister"</span> href=<span class="string">"http://example.com/lacie"</span> id=<span class="string">"link2"</span>&gt;<span class="type">Lacie</span>&lt;/a&gt;</span><br><span class="line">&lt;/p&gt;</span><br><span class="line">[<span class="symbol">'stor</span>y']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果返回结果是单个节点，那么可以直接调用<code>string</code>、<code>attrs</code>等属性获得其文本和属性；如果返回结果是多个节点的生成器，则可以转为列表后取出某个元素，然后再调用<code>string</code>、<code>attrs</code>等属性获取其对应节点的文本和属性。</p>
                  <h2 id="6-方法选择器"><a href="#6-方法选择器" class="headerlink" title="6. 方法选择器"></a>6. 方法选择器</h2>
                  <p>前面所讲的选择方法都是通过属性来选择的，这种方法非常快，但是如果进行比较复杂的选择的话，它就比较烦琐，不够灵活了。幸好，Beautiful Soup还为我们提供了一些查询方法，比如<code>find_all()</code>和<code>find()</code>等，调用它们，然后传入相应的参数，就可以灵活查询了。</p>
                  <h5 id="find-all"><a href="#find-all" class="headerlink" title="find_all()"></a><code>find_all()</code></h5>
                  <p><code>find_all</code>，顾名思义，就是查询所有符合条件的元素。给它传入一些属性或文本，就可以得到符合条件的元素，它的功能十分强大。</p>
                  <p>它的API如下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">find<span class="constructor">_all(<span class="params">name</span> , <span class="params">attrs</span> , <span class="params">recursive</span> , <span class="params">text</span> , <span class="operator">**</span><span class="params">kwargs</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="1-name"><a href="#1-name" class="headerlink" title="(1)name"></a>(1)<code>name</code></h3>
                  <p>我们可以根据节点名来查询元素，示例如下：</p>
                  <figure class="highlight scala">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html='''</span><br><span class="line">&lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel"</span>&gt;</span><br><span class="line">    &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel-heading"</span>&gt;</span><br><span class="line">        &lt;h4&gt;<span class="type">Hello</span>&lt;/h4&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">    &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel-body"</span>&gt;</span><br><span class="line">        &lt;ul <span class="class"><span class="keyword">class</span></span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Foo</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Bar</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Jay</span>&lt;/li&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">        &lt;ul <span class="class"><span class="keyword">class</span></span>=<span class="string">"list list-small"</span> id=<span class="string">"list-2"</span>&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Foo</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Bar</span>&lt;/li&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">&lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from bs4 <span class="keyword">import</span> <span class="type">BeautifulSoup</span></span><br><span class="line">soup = <span class="type">BeautifulSoup</span>(html, <span class="symbol">'lxm</span>l')</span><br><span class="line">print(soup.find_all(name=<span class="symbol">'u</span>l'))</span><br><span class="line">print(<span class="class"><span class="keyword">type</span>(<span class="params">soup.find_all(name='ul'</span>)[0]))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;, &lt;ul <span class="keyword">class</span>=<span class="string">"list list-small"</span> id=<span class="string">"list-2"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;]</span><br><span class="line">&lt;<span class="keyword">class</span> 'bs4.element.Tag'&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们调用了<code>find_all()</code>方法，传入<code>name</code>参数，其参数值为<code>ul</code>。也就是说，我们想要查询所有<code>ul</code>节点，返回结果是列表类型，长度为2，每个元素依然都是<code>bs4.element.Tag</code>类型。</p>
                  <p>因为都是<code>Tag</code>类型，所以依然可以进行嵌套查询。还是同样的文本，这里查询出所有<code>ul</code>节点后，再继续查询其内部的<code>li</code>节点：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">for</span> ul <span class="keyword">in</span> soup.find_all(<span class="attribute">name</span>=<span class="string">'ul'</span>):</span><br><span class="line">    <span class="builtin-name">print</span>(ul.find_all(<span class="attribute">name</span>=<span class="string">'li'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;li <span class="keyword">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Foo</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Bar</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Jay</span>&lt;/<span class="symbol">li</span>&gt;]</span><br><span class="line">[&lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Foo</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Bar</span>&lt;/<span class="symbol">li</span>&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>返回结果是列表类型，列表中的每个元素依然还是<code>Tag</code>类型。</p>
                  <p>接下来，就可以遍历每个<code>li</code>，获取它的文本了：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">for</span> ul <span class="keyword">in</span> soup.find_all(<span class="attribute">name</span>=<span class="string">'ul'</span>):</span><br><span class="line">    <span class="builtin-name">print</span>(ul.find_all(<span class="attribute">name</span>=<span class="string">'li'</span>))</span><br><span class="line">    <span class="keyword">for</span> li <span class="keyword">in</span> ul.find_all(<span class="attribute">name</span>=<span class="string">'li'</span>):</span><br><span class="line">        <span class="builtin-name">print</span>(li.string)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;li <span class="keyword">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Foo</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Bar</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Jay</span>&lt;/<span class="symbol">li</span>&gt;]</span><br><span class="line"><span class="symbol">Foo</span></span><br><span class="line"><span class="symbol">Bar</span></span><br><span class="line"><span class="symbol">Jay</span></span><br><span class="line">[&lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Foo</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Bar</span>&lt;/<span class="symbol">li</span>&gt;]</span><br><span class="line"><span class="symbol">Foo</span></span><br><span class="line"><span class="symbol">Bar</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h3 id="2-attrs"><a href="#2-attrs" class="headerlink" title="(2)attrs"></a>(2)<code>attrs</code></h3>
                  <p>除了根据节点名查询，我们也可以传入一些属性来查询，示例如下：</p>
                  <figure class="highlight scala">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html='''</span><br><span class="line">&lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel"</span>&gt;</span><br><span class="line">    &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel-heading"</span>&gt;</span><br><span class="line">        &lt;h4&gt;<span class="type">Hello</span>&lt;/h4&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">    &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel-body"</span>&gt;</span><br><span class="line">        &lt;ul <span class="class"><span class="keyword">class</span></span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span> name=<span class="string">"elements"</span>&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Foo</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Bar</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Jay</span>&lt;/li&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">        &lt;ul <span class="class"><span class="keyword">class</span></span>=<span class="string">"list list-small"</span> id=<span class="string">"list-2"</span>&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Foo</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Bar</span>&lt;/li&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">&lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from bs4 <span class="keyword">import</span> <span class="type">BeautifulSoup</span></span><br><span class="line">soup = <span class="type">BeautifulSoup</span>(html, <span class="symbol">'lxm</span>l')</span><br><span class="line">print(soup.find_all(attrs=&#123;<span class="symbol">'i</span>d': <span class="symbol">'list</span><span class="number">-1</span>'&#125;))</span><br><span class="line">print(soup.find_all(attrs=&#123;<span class="symbol">'nam</span>e': <span class="symbol">'element</span>s'&#125;))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span> name=<span class="string">"elements"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;]</span><br><span class="line">[&lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span> name=<span class="string">"elements"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里查询的时候传入的是<code>attrs</code>参数，参数的类型是字典类型。比如，要查询<code>id</code>为<code>list-1</code>的节点，可以传入<code>attrs={&#39;id&#39;: &#39;list-1&#39;}</code>的查询条件，得到的结果是列表形式，包含的内容就是符合<code>id</code>为<code>list-1</code>的所有节点。在上面的例子中，符合条件的元素个数是1，所以结果是长度为1的列表。</p>
                  <p>对于一些常用的属性，比如<code>id</code>和<code>class</code>等，我们可以不用<code>attrs</code>来传递。比如，要查询<code>id</code>为<code>list-1</code>的节点，可以直接传入<code>id</code>这个参数。还是上面的文本，我们换一种方式来查询：</p>
                  <figure class="highlight lisp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(<span class="name">html</span>, 'lxml')</span><br><span class="line">print(<span class="name">soup</span>.find_all(<span class="name">id=</span>'list-1'))</span><br><span class="line">print(<span class="name">soup</span>.find_all(<span class="name">class_=</span>'element'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;]</span><br><span class="line">[&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;, &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;, &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;, &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;, &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里直接传入<code>id=&#39;list-1&#39;</code>，就可以查询<code>id</code>为<code>list-1</code>的节点元素了。而对于<code>class</code>来说，由于<code>class</code>在Python里是一个关键字，所以后面需要加一个下划线，即<code>class_=&#39;element&#39;</code>，返回的结果依然还是<code>Tag</code>组成的列表。</p>
                  <h3 id="3-text"><a href="#3-text" class="headerlink" title="(3)text"></a>(3)<code>text</code></h3>
                  <p><code>text</code>参数可用来匹配节点的文本，传入的形式可以是字符串，可以是正则表达式对象，示例如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> re</span><br><span class="line">html=<span class="string">'''</span></span><br><span class="line"><span class="string">&lt;div class="panel"&gt;</span></span><br><span class="line"><span class="string">    &lt;div class="panel-body"&gt;</span></span><br><span class="line"><span class="string">        &lt;a&gt;Hello, this is a link&lt;/a&gt;</span></span><br><span class="line"><span class="string">        &lt;a&gt;Hello, this is a link, too&lt;/a&gt;</span></span><br><span class="line"><span class="string">    &lt;/div&gt;</span></span><br><span class="line"><span class="string">&lt;/div&gt;</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line">print(soup.find_all(text=re.compile(<span class="string">'link'</span>)))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['Hello, this is a link', 'Hello, this is a link, too']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里有两个<code>a</code>节点，其内部包含文本信息。这里在<code>find_all()</code>方法中传入<code>text</code>参数，该参数为正则表达式对象，结果返回所有匹配正则表达式的节点文本组成的列表。</p>
                  <h4 id="find"><a href="#find" class="headerlink" title="find()"></a><code>find()</code></h4>
                  <p>除了<code>find_all()</code>方法，还有<code>find()</code>方法，只不过后者返回的是单个元素，也就是第一个匹配的元素，而前者返回的是所有匹配的元素组成的列表。示例如下：</p>
                  <figure class="highlight scala">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html='''</span><br><span class="line">&lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel"</span>&gt;</span><br><span class="line">    &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel-heading"</span>&gt;</span><br><span class="line">        &lt;h4&gt;<span class="type">Hello</span>&lt;/h4&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">    &lt;div <span class="class"><span class="keyword">class</span></span>=<span class="string">"panel-body"</span>&gt;</span><br><span class="line">        &lt;ul <span class="class"><span class="keyword">class</span></span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Foo</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Bar</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Jay</span>&lt;/li&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">        &lt;ul <span class="class"><span class="keyword">class</span></span>=<span class="string">"list list-small"</span> id=<span class="string">"list-2"</span>&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Foo</span>&lt;/li&gt;</span><br><span class="line">            &lt;li <span class="class"><span class="keyword">class</span></span>=<span class="string">"element"</span>&gt;<span class="type">Bar</span>&lt;/li&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">&lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from bs4 <span class="keyword">import</span> <span class="type">BeautifulSoup</span></span><br><span class="line">soup = <span class="type">BeautifulSoup</span>(html, <span class="symbol">'lxm</span>l')</span><br><span class="line">print(soup.find(name=<span class="symbol">'u</span>l'))</span><br><span class="line">print(<span class="class"><span class="keyword">type</span>(<span class="params">soup.find(name='ul'</span>)))</span></span><br><span class="line"><span class="class"><span class="title">print</span>(<span class="params">soup.find(class_='list'</span>))</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;</span><br><span class="line">&lt;<span class="keyword">class</span> 'bs4.element.Tag'&gt;</span><br><span class="line">&lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">&lt;/ul&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的返回结果不再是列表形式，而是第一个匹配的节点元素，类型依然是<code>Tag</code>类型。</p>
                  <p>另外，还有许多查询方法，其用法与前面介绍的<code>find_all()</code>、<code>find()</code>方法完全相同，只不过查询范围不同，这里简单说明一下。</p>
                  <ul>
                    <li><strong><code>find_parents()</code>和<code>find_parent()</code></strong>：前者返回所有祖先节点，后者返回直接父节点。</li>
                    <li><strong><code>find_next_siblings()</code>和<code>find_next_sibling()</code></strong>：前者返回后面所有的兄弟节点，后者返回后面第一个兄弟节点。</li>
                    <li><strong><code>find_previous_siblings()</code>和<code>find_previous_sibling()</code></strong>：前者返回前面所有的兄弟节点，后者返回前面第一个兄弟节点。</li>
                    <li><strong><code>find_all_next()</code>和<code>find_next()</code></strong>：前者返回节点后所有符合条件的节点，后者返回第一个符合条件的节点。</li>
                    <li><strong><code>find_all_previous()</code>和<code>find_previous()</code></strong>：前者返回节点后所有符合条件的节点，后者返回第一个符合条件的节点。</li>
                  </ul>
                  <h2 id="7-CSS选择器"><a href="#7-CSS选择器" class="headerlink" title="7. CSS选择器"></a>7. CSS选择器</h2>
                  <p>Beautiful Soup还提供了另外一种选择器，那就是CSS选择器。如果对Web开发熟悉的话，那么对CSS选择器肯定也不陌生。如果不熟悉的话，可以参考<a href="http://www.w3school.com.cn/cssref/css_selectors.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/cssref/css_selectors.asp</a>了解。</p>
                  <p>使用CSS选择器时，只需要调用<code>select()</code>方法，传入相应的CSS选择器即可，示例如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html='''</span><br><span class="line">&lt;div <span class="keyword">class</span>=<span class="string">"panel"</span>&gt;</span><br><span class="line">    &lt;div <span class="keyword">class</span>=<span class="string">"panel-heading"</span>&gt;</span><br><span class="line">        &lt;h4&gt;Hello&lt;/h4&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">    &lt;div <span class="keyword">class</span>=<span class="string">"panel-body"</span>&gt;</span><br><span class="line">        &lt;ul <span class="keyword">class</span>=<span class="string">"list"</span> id=<span class="string">"list-1"</span>&gt;</span><br><span class="line">            &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">            &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">            &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Jay&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">        &lt;ul <span class="keyword">class</span>=<span class="string">"list list-small"</span> id=<span class="string">"list-2"</span>&gt;</span><br><span class="line">            &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Foo&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">            &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"element"</span>&gt;Bar&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;/ul&gt;</span><br><span class="line">    &lt;/div&gt;</span><br><span class="line">&lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">from bs4 import BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, 'lxml')</span><br><span class="line"><span class="keyword">print</span>(soup.select('.panel .panel-heading'))</span><br><span class="line"><span class="keyword">print</span>(soup.select('ul <span class="keyword">li</span>'))</span><br><span class="line"><span class="keyword">print</span>(soup.select('#<span class="keyword">list</span>-2 .element'))</span><br><span class="line"><span class="keyword">print</span>(<span class="keyword">type</span>(soup.select('ul')[0]))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;div class="panel-heading"&gt;</span></span><br><span class="line"><span class="meta">&lt;h4&gt;Hello&lt;/h4&gt;</span></span><br><span class="line"><span class="meta">&lt;/div&gt;]</span></span><br><span class="line"><span class="meta">[&lt;li class="element"&gt;Foo&lt;/li&gt;, &lt;li class="element"&gt;Bar&lt;/li&gt;, &lt;li class="element"&gt;Jay&lt;/li&gt;, &lt;li class="element"&gt;Foo&lt;/li&gt;, &lt;li class="element"&gt;Bar&lt;/li&gt;]</span></span><br><span class="line"><span class="meta">[&lt;li class="element"&gt;Foo&lt;/li&gt;, &lt;li class="element"&gt;Bar&lt;/li&gt;]</span></span><br><span class="line">&lt;<span class="keyword">class</span> 'bs4.element.Tag'&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们用了3次CSS选择器，返回的结果均是符合CSS选择器的节点组成的列表。例如，<code>select(&#39;ul li&#39;)</code>则是选择所有<code>ul</code>节点下面的所有<code>li</code>节点，结果便是所有的<code>li</code>节点组成的列表。</p>
                  <p>最后一句打印输出了列表中元素的类型。可以看到，类型依然是<code>Tag</code>类型。</p>
                  <h3 id="嵌套选择-1"><a href="#嵌套选择-1" class="headerlink" title="嵌套选择"></a>嵌套选择</h3>
                  <p><code>select()</code>方法同样支持嵌套选择。例如，先选择所有<code>ul</code>节点，再遍历每个<code>ul</code>节点，选择其<code>li</code>节点，样例如下：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="keyword">for</span> ul <span class="keyword">in</span> soup.<span class="built_in">select</span>(<span class="string">'ul'</span>):</span><br><span class="line">    <span class="built_in">print</span>(ul.<span class="built_in">select</span>(<span class="string">'li'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;li <span class="keyword">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Foo</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Bar</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Jay</span>&lt;/<span class="symbol">li</span>&gt;]</span><br><span class="line">[&lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Foo</span>&lt;/<span class="symbol">li</span>&gt;, &lt;<span class="symbol">li</span> <span class="symbol">class</span>="<span class="symbol">element</span>"&gt;<span class="symbol">Bar</span>&lt;/<span class="symbol">li</span>&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里正常输出了所有<code>ul</code>节点下所有<code>li</code>节点组成的列表。</p>
                  <h3 id="获取属性"><a href="#获取属性" class="headerlink" title="获取属性"></a>获取属性</h3>
                  <p>我们知道节点类型是<code>Tag</code>类型，所以获取属性还可以用原来的方法。仍然是上面的HTML文本，这里尝试获取每个<code>ul</code>节点的<code>id</code>属性：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="keyword">for</span> ul <span class="keyword">in</span> soup.<span class="built_in">select</span>(<span class="string">'ul'</span>):</span><br><span class="line">    <span class="built_in">print</span>(ul[<span class="string">'id'</span>])</span><br><span class="line">    <span class="built_in">print</span>(ul.attrs[<span class="string">'id'</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight dsconfig">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="built_in">list-1</span></span><br><span class="line"><span class="built_in">list-1</span></span><br><span class="line"><span class="built_in">list-2</span></span><br><span class="line"><span class="built_in">list-2</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，直接传入中括号和属性名，以及通过<code>attrs</code>属性获取属性值，都可以成功。</p>
                  <h3 id="获取文本"><a href="#获取文本" class="headerlink" title="获取文本"></a>获取文本</h3>
                  <p>要获取文本，当然也可以用前面所讲的<code>string</code>属性。此外，还有一个方法，那就是<code>get_text()</code>，示例如下：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> bs4 <span class="keyword">import</span> BeautifulSoup</span><br><span class="line">soup = BeautifulSoup(html, <span class="string">'lxml'</span>)</span><br><span class="line"><span class="keyword">for</span> li <span class="keyword">in</span> soup.<span class="built_in">select</span>(<span class="string">'li'</span>):</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">'Get Text:'</span>, li.get_text())</span><br><span class="line">    <span class="built_in">print</span>(<span class="string">'String:'</span>, li.<span class="built_in">string</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight mathematica">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">Get</span> <span class="keyword">Text</span>: Foo</span><br><span class="line"><span class="keyword">String</span>: Foo</span><br><span class="line"><span class="keyword">Get</span> <span class="keyword">Text</span>: Bar</span><br><span class="line"><span class="keyword">String</span>: Bar</span><br><span class="line"><span class="keyword">Get</span> <span class="keyword">Text</span>: Jay</span><br><span class="line"><span class="keyword">String</span>: Jay</span><br><span class="line"><span class="keyword">Get</span> <span class="keyword">Text</span>: Foo</span><br><span class="line"><span class="keyword">String</span>: Foo</span><br><span class="line"><span class="keyword">Get</span> <span class="keyword">Text</span>: Bar</span><br><span class="line"><span class="keyword">String</span>: Bar</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，二者的效果完全一致。</p>
                  <p>到此，Beautiful Soup的用法基本就介绍完了，最后做一下简单的总结。</p>
                  <ul>
                    <li>推荐使用lxml解析库，必要时使用html.parser。</li>
                    <li>节点选择筛选功能弱但是速度快。</li>
                    <li>建议使用<code>find()</code>或者<code>find_all()</code>查询匹配单个结果或者多个结果。</li>
                    <li>如果对CSS选择器熟悉的话，可以使用<code>select()</code>方法选择。</li>
                  </ul>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-28 16:19:52" itemprop="dateCreated datePublished" datetime="2018-01-28T16:19:52+08:00">2018-01-28</time>
                </span>
                <span id="/5548.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 4.2-使用Beautiful Soup" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>22k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>20 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5545.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5545.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 4.1-使用XPath</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>XPath，全称XML Path Language，即XML路径语言，它是一门在XML文档中查找信息的语言。它最初是用来搜寻XML文档的，但是它同样适用于HTML文档的搜索。</p>
                  <p>所以在做爬虫时，我们完全可以使用XPath来做相应的信息抽取。本节中，我们就来介绍XPath的基本用法。</p>
                  <h2 id="1-XPath概览"><a href="#1-XPath概览" class="headerlink" title="1. XPath概览"></a>1. XPath概览</h2>
                  <p>XPath的选择功能十分强大，它提供了非常简洁明了的路径选择表达式。另外，它还提供了超过100个内建函数，用于字符串、数值、时间的匹配以及节点、序列的处理等。几乎所有我们想要定位的节点，都可以用XPath来选择。</p>
                  <p>XPath于1999年11月16日成为W3C标准，它被设计为供XSLT、XPointer以及其他XML解析软件使用，更多的文档可以访问其官方网站：<a href="https://www.w3.org/TR/xpath/" target="_blank" rel="noopener">https://www.w3.org/TR/xpath/</a>。</p>
                  <h2 id="2-XPath常用规则"><a href="#2-XPath常用规则" class="headerlink" title="2. XPath常用规则"></a>2. XPath常用规则</h2>
                  <p>表4-1列举了XPath的几个常用规则。</p>
                  <p>表4-1 XPath常用规则</p>
                  <p>表达式</p>
                  <p>描述</p>
                  <p><code>nodename</code></p>
                  <p>选取此节点的所有子节点</p>
                  <p><code>/</code></p>
                  <p>从当前节点选取直接子节点</p>
                  <p><code>//</code></p>
                  <p>从当前节点选取子孙节点</p>
                  <p><code>.</code></p>
                  <p>选取当前节点</p>
                  <p><code>..</code></p>
                  <p>选取当前节点的父节点</p>
                  <p><code>@</code></p>
                  <p>选取属性</p>
                  <p>这里列出了XPath的常用匹配规则，示例如下：</p>
                  <figure class="highlight autoit">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">//title[<span class="symbol">@lang</span>=<span class="string">'eng'</span>]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这就是一个XPath规则，它代表选择所有名称为<code>title</code>，同时属性<code>lang</code>的值为<code>eng</code>的节点。</p>
                  <p>后面会通过Python的lxml库，利用XPath进行HTML的解析。</p>
                  <h2 id="3-准备工作"><a href="#3-准备工作" class="headerlink" title="3. 准备工作"></a>3. 准备工作</h2>
                  <p>使用之前，首先要确保安装好lxml库，若没有安装，可以参考第1章的安装过程。</p>
                  <h2 id="4-实例引入"><a href="#4-实例引入" class="headerlink" title="4. 实例引入"></a>4. 实例引入</h2>
                  <p>现在通过实例来感受一下使用XPath来对网页进行解析的过程，相关代码如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line">text = '''</span><br><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line">'''</span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = etree.tostring(html)</span><br><span class="line">print(result.decode('utf-8'))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先导入lxml库的etree模块，然后声明了一段HTML文本，调用HTML类进行初始化，这样就成功构造了一个XPath解析对象。这里需要注意的是，HTML文本中的最后一个<code>li</code>节点是没有闭合的，但是etree模块可以自动修正HTML文本。</p>
                  <p>这里我们调用<code>tostring()</code>方法即可输出修正后的HTML代码，但是结果是<code>bytes</code>类型。这里利用<code>decode()</code>方法将其转成<code>str</code>类型，结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span><span class="tag">&lt;<span class="name">body</span>&gt;</span><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">li</span>&gt;</span><span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，经过处理之后，<code>li</code>节点标签被补全，并且还自动添加了<code>body</code>、<code>html</code>节点。</p>
                  <p>另外，也可以直接读取文本文件进行解析，示例如下：</p>
                  <figure class="highlight moonscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"></span><br><span class="line">html = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = etree.<span class="built_in">tostring</span>(html)</span><br><span class="line"><span class="built_in">print</span>(result.decode(<span class="string">'utf-8'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中test.html的内容就是上面例子中的HTML代码，内容如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这次的输出结果略有不同，多了一个<code>DOCTYPE</code>的声明，不过对解析无任何影响，结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">&lt;!DOCTYPE <span class="meta-keyword">html</span> <span class="meta-keyword">PUBLIC</span> <span class="meta-string">"-//W3C//DTD HTML 4.0 Transitional//EN"</span> <span class="meta-string">"http://www.w3.org/TR/REC-html40/loose.dtd"</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span><span class="tag">&lt;<span class="name">body</span>&gt;</span><span class="tag">&lt;<span class="name">div</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link2.html"</span>&gt;</span>second item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-inactive"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link3.html"</span>&gt;</span>third item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-1"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link4.html"</span>&gt;</span>fourth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">         <span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">     <span class="tag">&lt;/<span class="name">li</span>&gt;</span><span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"> <span class="tag">&lt;/<span class="name">div</span>&gt;</span><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="5-所有节点"><a href="#5-所有节点" class="headerlink" title="5. 所有节点"></a>5. 所有节点</h2>
                  <p>我们一般会用//开头的XPath规则来选取所有符合要求的节点。这里以前面的HTML文本为例，如果要选取所有节点，可以这样实现：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//*'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight x86asm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;Element html <span class="meta">at</span> <span class="number">0x10510d9c8</span>&gt;, &lt;Element body <span class="meta">at</span> <span class="number">0x10510da08</span>&gt;, &lt;Element <span class="keyword">div</span> <span class="meta">at</span> <span class="number">0x10510da48</span>&gt;, &lt;Element ul <span class="meta">at</span> <span class="number">0x10510da88</span>&gt;, &lt;Element li <span class="meta">at</span> <span class="number">0x10510dac8</span>&gt;, &lt;Element a <span class="meta">at</span> <span class="number">0x10510db48</span>&gt;, &lt;Element li <span class="meta">at</span> <span class="number">0x10510db88</span>&gt;, &lt;Element a <span class="meta">at</span> <span class="number">0x10510dbc8</span>&gt;, &lt;Element li <span class="meta">at</span> <span class="number">0x10510dc08</span>&gt;, &lt;Element a <span class="meta">at</span> <span class="number">0x10510db08</span>&gt;, &lt;Element li <span class="meta">at</span> <span class="number">0x10510dc48</span>&gt;, &lt;Element a <span class="meta">at</span> <span class="number">0x10510dc88</span>&gt;, &lt;Element li <span class="meta">at</span> <span class="number">0x10510dcc8</span>&gt;, &lt;Element a <span class="meta">at</span> <span class="number">0x10510dd08</span>&gt;]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里使用*代表匹配所有节点，也就是整个HTML文本中的所有节点都会被获取。可以看到，返回形式是一个列表，每个元素是<code>Element</code>类型，其后跟了节点的名称，如<code>html</code>、<code>body</code>、<code>div</code>、<code>ul</code>、<code>li</code>、<code>a</code>等，所有节点都包含在列表中了。</p>
                  <p>当然，此处匹配也可以指定节点名称。如果想获取所有<code>li</code>节点，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result[<span class="number">0</span>])</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里要选取所有<code>li</code>节点，可以使用<code>//</code>，然后直接加上节点名称即可，调用时直接使用<code>xpath()</code>方法即可。</p>
                  <p>运行结果：</p>
                  <figure class="highlight maxima">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[&lt;Element <span class="built_in">li</span> <span class="built_in">at</span> <span class="number">0x105849208</span>&gt;, &lt;Element <span class="built_in">li</span> <span class="built_in">at</span> <span class="number">0x105849248</span>&gt;, &lt;Element <span class="built_in">li</span> <span class="built_in">at</span> <span class="number">0x105849288</span>&gt;, &lt;Element <span class="built_in">li</span> <span class="built_in">at</span> <span class="number">0x1058492c8</span>&gt;, &lt;Element <span class="built_in">li</span> <span class="built_in">at</span> <span class="number">0x105849308</span>&gt;]</span><br><span class="line">&lt;Element <span class="built_in">li</span> <span class="built_in">at</span> <span class="number">0x105849208</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里可以看到提取结果是一个列表形式，其中每个元素都是一个 <code>Element</code>对象。如果要取出其中一个对象，可以直接用中括号加索引，如<code>[0]</code>。</p>
                  <h2 id="6-子节点"><a href="#6-子节点" class="headerlink" title="6. 子节点"></a>6. 子节点</h2>
                  <p>我们通过<code>/</code>或<code>//</code>即可查找元素的子节点或子孙节点。假如现在想选择<code>li</code>节点的所有直接<code>a</code>子节点，可以这样实现：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li/a'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里通过追加<code>/a</code>即选择了所有<code>li</code>节点的所有直接<code>a</code>子节点。因为<code>//li</code>用于选中所有<code>li</code>节点，<code>/a</code>用于选中<code>li</code>节点的所有直接子节点<code>a</code>，二者组合在一起即获取所有<code>li</code>节点的所有直接<code>a</code>子节点。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Element a at 0x106ee8688&gt;, &lt;Element a at 0x106ee86c8&gt;, &lt;Element a at 0x106ee8708&gt;, &lt;Element a at 0x106ee8748&gt;, &lt;Element a at 0x106ee8788&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此处的<code>/</code>用于选取直接子节点，如果要获取所有子孙节点，就可以使用<code>//</code>。例如，要获取<code>ul</code>节点下的所有子孙<code>a</code>节点，可以这样实现：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//ul//a'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果是相同的。</p>
                  <p>但是如果这里用<code>//ul/a</code>，就无法获取任何结果了。因为<code>/</code>用于获取直接子节点，而在<code>ul</code>节点下没有直接的<code>a</code>子节点，只有<code>li</code>节点，所以无法获取任何匹配结果，代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//ul/a'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因此，这里我们要注意<code>/</code>和<code>//</code>的区别，其中<code>/</code>用于获取直接子节点，<code>//</code>用于获取子孙节点。</p>
                  <h2 id="7-父节点"><a href="#7-父节点" class="headerlink" title="7. 父节点"></a>7. 父节点</h2>
                  <p>我们知道通过连续的<code>/</code>或<code>//</code>可以查找子节点或子孙节点，那么假如我们知道了子节点，怎样来查找父节点呢？这可以用<code>..</code>来实现。</p>
                  <p>比如，现在首先选中<code>href</code>属性为<code>link4.html</code>的<code>a</code>节点，然后再获取其父节点，然后再获取其<code>class</code>属性，相关代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//a[@href="link4.html"]/../@class'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'item-1</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>检查一下结果发现，这正是我们获取的目标<code>li</code>节点的<code>class</code>。</p>
                  <p>同时，我们也可以通过<code>parent::</code>来获取父节点，代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//a[@href="link4.html"]/parent::*/@class'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="8-属性匹配"><a href="#8-属性匹配" class="headerlink" title="8. 属性匹配"></a>8. 属性匹配</h2>
                  <p>在选取的时候，我们还可以用<code>@</code>符号进行属性过滤。比如，这里如果要选取<code>class</code>为<code>item-1</code>的<code>li</code>节点，可以这样实现:</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li[@class="item-0"]'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们通过加入<code>[@class=&quot;item-0&quot;]</code>，限制了节点的<code>class</code>属性为<code>item-0</code>，而HTML文本中符合条件的<code>li</code>节点有两个，所以结果应该返回两个匹配到的元素。结果如下：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Element li at 0x10a399288&gt;, &lt;Element li at 0x10a3992c8&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可见，匹配结果正是两个，至于是不是那正确的两个，后面再验证。</p>
                  <h2 id="9-文本获取"><a href="#9-文本获取" class="headerlink" title="9. 文本获取"></a>9. 文本获取</h2>
                  <p>我们用XPath中的<code>text()</code>方法获取节点中的文本，接下来尝试获取前面<code>li</code>节点中的文本，相关代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li[@class="item-0"]/text()'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['\n     ']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>奇怪的是，我们并没有获取到任何文本，只获取到了一个换行符，这是为什么呢？因为XPath中<code>text()</code>前面是<code>/</code>，而此处<code>/</code>的含义是选取直接子节点，很明显<code>li</code>的直接子节点都是<code>a</code>节点，文本都是在<code>a</code>节点内部的，所以这里匹配到的结果就是被修正的<code>li</code>节点内部的换行符，因为自动修正的<code>li</code>节点的尾标签换行了。</p>
                  <p>即选中的是这两个节点：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link1.html"</span>&gt;</span>first item<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;<span class="name">li</span> <span class="attr">class</span>=<span class="string">"item-0"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"link5.html"</span>&gt;</span>fifth item<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>其中一个节点因为自动修正，<code>li</code>节点的尾标签添加的时候换行了，所以提取文本得到的唯一结果就是<code>li</code>节点的尾标签和<code>a</code>节点的尾标签之间的换行符。</p>
                  <p>因此，如果想获取<code>li</code>节点内部的文本，就有两种方式，一种是先选取<code>a</code>节点再获取文本，另一种就是使用<code>//</code>。接下来，我们来看下二者的区别。</p>
                  <p>首先，选取到<code>a</code>节点再获取文本，代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li[@class="item-0"]/a/text()'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'first</span> item', <span class="symbol">'fifth</span> item']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里的返回值是两个，内容都是属性为<code>item-0</code>的<code>li</code>节点的文本，这也印证了前面属性匹配的结果是正确的。</p>
                  <p>这里我们是逐层选取的，先选取了<code>li</code>节点，又利用<code>/</code>选取了其直接子节点<code>a</code>，然后再选取其文本，得到的结果恰好是符合我们预期的两个结果。</p>
                  <p>再来看下用另一种方式（即使用<code>//</code>）选取的结果，代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li[@class="item-0"]//text()'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'first</span> item', <span class="symbol">'fifth</span> item', '\n     ']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不出所料，这里的返回结果是3个。可想而知，这里是选取所有子孙节点的文本，其中前两个就是<code>li</code>的子节点<code>a</code>节点内部的文本，另外一个就是最后一个<code>li</code>节点内部的文本，即换行符。</p>
                  <p>所以说，如果要想获取子孙节点内部的所有文本，可以直接用<code>//</code>加<code>text()</code>的方式，这样可以保证获取到最全面的文本信息，但是可能会夹杂一些换行符等特殊字符。如果想获取某些特定子孙节点下的所有文本，可以先选取到特定的子孙节点，然后再调用<code>text()</code>方法获取其内部文本，这样可以保证获取的结果是整洁的。</p>
                  <h2 id="10-属性获取"><a href="#10-属性获取" class="headerlink" title="10. 属性获取"></a>10. 属性获取</h2>
                  <p>我们知道用<code>text()</code>可以获取节点内部文本，那么节点属性该怎样获取呢？其实还是用<code>@</code>符号就可以。例如，我们想获取所有<code>li</code>节点下所有<code>a</code>节点的<code>href</code>属性，代码如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line"><span class="selector-tag">html</span> = etree.parse(<span class="string">'./test.html'</span>, etree.HTMLParser())</span><br><span class="line">result = <span class="selector-tag">html</span>.xpath(<span class="string">'//li/a/@href'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们通过<code>@href</code>即可获取节点的<code>href</code>属性。注意，此处和属性匹配的方法不同，属性匹配是中括号加属性名和值来限定某个属性，如<code>[@href=&quot;link1.html&quot;]</code>，而此处的<code>@href</code>指的是获取节点的某个属性，二者需要做好区分。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'link1.html</span>', <span class="symbol">'link2.html</span>', <span class="symbol">'link3.html</span>', <span class="symbol">'link4.html</span>', <span class="symbol">'link5.html</span>']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，我们成功获取了所有<code>li</code>节点下<code>a</code>节点的<code>href</code>属性，它们以列表形式返回。</p>
                  <h2 id="11-属性多值匹配"><a href="#11-属性多值匹配" class="headerlink" title="11. 属性多值匹配"></a>11. 属性多值匹配</h2>
                  <p>有时候，某些节点的某个属性可能有多个值，例如：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line">text = <span class="string">'''</span></span><br><span class="line"><span class="string">&lt;li class="li li-first"&gt;&lt;a href="link.html"&gt;first item&lt;/a&gt;&lt;/li&gt;</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = html.xpath(<span class="string">'//li[@class="li"]/a/text()'</span>)</span><br><span class="line">print(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里HTML文本中<code>li</code>节点的<code>class</code>属性有两个值<code>li</code>和<code>li-first</code>，此时如果还想用之前的属性匹配获取，就无法匹配了，此时的运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时就需要用<code>contains()</code>函数了，代码可以改写如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line">text = <span class="string">'''</span></span><br><span class="line"><span class="string">&lt;li class="li li-first"&gt;&lt;a href="link.html"&gt;first item&lt;/a&gt;&lt;/li&gt;</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = html.xpath(<span class="string">'//li[contains(@class, "li")]/a/text()'</span>)</span><br><span class="line">print(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样通过<code>contains()</code>方法，第一个参数传入属性名称，第二个参数传入属性值，只要此属性包含所传入的属性值，就可以完成匹配了。</p>
                  <p>此时运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'first</span> item']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此种方式在某个节点的某个属性有多个值时经常用到，如某个节点的<code>class</code>属性通常有多个。</p>
                  <h2 id="12-多属性匹配"><a href="#12-多属性匹配" class="headerlink" title="12. 多属性匹配"></a>12. 多属性匹配</h2>
                  <p>另外，我们可能还遇到一种情况，那就是根据多个属性确定一个节点，这时就需要同时匹配多个属性。此时可以使用运算符<code>and</code>来连接，示例如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line">text = <span class="string">'''</span></span><br><span class="line"><span class="string">&lt;li class="li li-first" name="item"&gt;&lt;a href="link.html"&gt;first item&lt;/a&gt;&lt;/li&gt;</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = html.xpath(<span class="string">'//li[contains(@class, "li") and @name="item"]/a/text()'</span>)</span><br><span class="line">print(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的<code>li</code>节点又增加了一个属性<code>name</code>。要确定这个节点，需要同时根据<code>class</code>和<code>name</code>属性来选择，一个条件是<code>class</code>属性里面包含<code>li</code>字符串，另一个条件是<code>name</code>属性为<code>item</code>字符串，二者需要同时满足，需要用<code>and</code>操作符相连，相连之后置于中括号内进行条件筛选。运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'first</span> item']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的<code>and</code>其实是XPath中的运算符。另外，还有很多运算符，如<code>or</code>、<code>mod</code>等，在此总结为表4-2。</p>
                  <p>表4-2 运算符及其介绍</p>
                  <p>运算符</p>
                  <p>描述</p>
                  <p>实例</p>
                  <p>返回值</p>
                  <p><code>or</code></p>
                  <p>或</p>
                  <p><code>age=19 or age=20</code></p>
                  <p>如果<code>age</code>是19，则返回<code>true</code>。如果<code>age</code>是21，则返回<code>false</code></p>
                  <p><code>and</code></p>
                  <p>与</p>
                  <p><code>age&gt;19 and age&lt;21</code></p>
                  <p>如果<code>age</code>是20，则返回<code>true</code>。如果<code>age</code>是<code>18</code>，则返回<code>false</code></p>
                  <p><code>mod</code></p>
                  <p>计算除法的余数</p>
                  <p><code>5 mod 2</code></p>
                  <p>1</p>
                  <p><code>|</code></p>
                  <p>计算两个节点集</p>
                  <p><code>//book | //cd</code></p>
                  <p>返回所有拥有<code>book</code>和<code>cd</code>元素的节点集</p>
                  <p><code>+</code></p>
                  <p>加法</p>
                  <p><code>6 + 4</code></p>
                  <p>10</p>
                  <p><code>\-</code></p>
                  <p>减法</p>
                  <p><code>6 - 4</code></p>
                  <p>2</p>
                  <p><code>*</code></p>
                  <p>乘法</p>
                  <p><code>6 * 4</code></p>
                  <p>24</p>
                  <p><code>div</code></p>
                  <p>除法</p>
                  <p><code>8 div 4</code></p>
                  <p>2</p>
                  <p><code>\=</code></p>
                  <p>等于</p>
                  <p><code>age=19</code></p>
                  <p>如果<code>age</code>是19，则返回<code>true</code>。如果<code>age</code>是20，则返回<code>false</code></p>
                  <p><code>!=</code></p>
                  <p>不等于</p>
                  <p><code>age!=19</code></p>
                  <p>如果<code>age</code>是18，则返回<code>true</code>。如果<code>age</code>是19，则返回<code>false</code></p>
                  <p><code>&lt;</code></p>
                  <p>小于</p>
                  <p><code>age&lt;19</code></p>
                  <p>如果<code>age</code>是18，则返回<code>true</code>。如果<code>age</code>是19，则返回<code>false</code></p>
                  <p><code>&lt;=</code></p>
                  <p>小于或等于</p>
                  <p><code>age&lt;=19</code></p>
                  <p>如果<code>age</code>是19，则返回<code>true</code>。如果<code>age</code>是20，则返回<code>false</code></p>
                  <p><code>\&gt;</code></p>
                  <p>大于</p>
                  <p><code>age&gt;19</code></p>
                  <p>如果<code>age</code>是20，则返回<code>true</code>。如果<code>age</code>是19，则返回<code>false</code></p>
                  <p><code>\&gt;=</code></p>
                  <p>大于或等于</p>
                  <p><code>age&gt;=19</code></p>
                  <p>如果<code>age</code>是19，则返回<code>true</code>。如果<code>age</code>是18，则返回<code>false</code></p>
                  <p>此表参考来源：<a href="http://www.w3school.com.cn/xpath/xpath_operators.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/xpath/xpath_operators.asp</a>。</p>
                  <h2 id="13-按序选择"><a href="#13-按序选择" class="headerlink" title="13. 按序选择"></a>13. 按序选择</h2>
                  <p>有时候，我们在选择的时候某些属性可能同时匹配了多个节点，但是只想要其中的某个节点，如第二个节点或者最后一个节点，这时该怎么办呢？</p>
                  <p>这时可以利用中括号传入索引的方法获取特定次序的节点，示例如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line">text = '''</span><br><span class="line">&lt;div&gt;</span><br><span class="line">    &lt;ul&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link1.html"</span>&gt;first item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;second item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-inactive"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;third item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;fourth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;fifth item&lt;/a&gt;</span><br><span class="line">     &lt;/ul&gt;</span><br><span class="line"> &lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/a/text()')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[last()]/a/text()')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[position()&lt;3]/a/text()')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[last()-2]/a/text()')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一次选择时，我们选取了第一个<code>li</code>节点，中括号中传入数字1即可。注意，这里和代码中不同，序号是以1开头的，不是以0开头。</p>
                  <p>第二次选择时，我们选取了最后一个<code>li</code>节点，中括号中传入<code>last()</code>即可，返回的便是最后一个<code>li</code>节点。</p>
                  <p>第三次选择时，我们选取了位置小于3的<code>li</code>节点，也就是位置序号为1和2的节点，得到的结果就是前两个<code>li</code>节点。</p>
                  <p>第四次选择时，我们选取了倒数第三个<code>li</code>节点，中括号中传入<code>last()-2</code>即可。因为<code>last()</code>是最后一个，所以<code>last()-2</code>就是倒数第三个。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="symbol">'first</span> item']</span><br><span class="line">[<span class="symbol">'fifth</span> item']</span><br><span class="line">[<span class="symbol">'first</span> item', <span class="symbol">'second</span> item']</span><br><span class="line">[<span class="symbol">'third</span> item']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们使用了<code>last()</code>、<code>position()</code>等函数。在XPath中，提供了100多个函数，包括存取、数值、字符串、逻辑、节点、序列等处理功能，它们的具体作用可以参考：<a href="http://www.w3school.com.cn/xpath/xpath_functions.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/xpath/xpath_functions.asp</a>。</p>
                  <h2 id="14-节点轴选择"><a href="#14-节点轴选择" class="headerlink" title="14. 节点轴选择"></a>14. 节点轴选择</h2>
                  <p>XPath提供了很多节点轴选择方法，包括获取子元素、兄弟元素、父元素、祖先元素等，示例如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from lxml import etree</span><br><span class="line"></span><br><span class="line">text = '''</span><br><span class="line">&lt;div&gt;</span><br><span class="line">    &lt;ul&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link1.html"</span>&gt;&lt;span&gt;first item&lt;/span&gt;&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link2.html"</span>&gt;second item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-inactive"</span>&gt;&lt;a href=<span class="string">"link3.html"</span>&gt;third item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-1"</span>&gt;&lt;a href=<span class="string">"link4.html"</span>&gt;fourth item&lt;/a&gt;&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">         &lt;<span class="keyword">li</span> <span class="keyword">class</span>=<span class="string">"item-0"</span>&gt;&lt;a href=<span class="string">"link5.html"</span>&gt;fifth item&lt;/a&gt;</span><br><span class="line">     &lt;/ul&gt;</span><br><span class="line"> &lt;/div&gt;</span><br><span class="line">'''</span><br><span class="line">html = etree.HTML(text)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/ancestor::*')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/ancestor::div')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/attribute::*')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/child::a[@href="link1.html"]')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/descendant::span')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/following::*[2]')</span></span><br><span class="line"><span class="keyword">print</span>(result)</span><br><span class="line">result = html.xpath('<span class="comment">//li[1]/following-sibling::*') `print(result)`</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="meta">[&lt;Element html at 0x107941808&gt;, &lt;Element body at 0x1079418c8&gt;, &lt;Element div at 0x107941908&gt;, &lt;Element ul at 0x107941948&gt;]</span></span><br><span class="line"><span class="meta">[&lt;Element div at 0x107941908&gt;]</span></span><br><span class="line">['item<span class="number">-0</span>']</span><br><span class="line"><span class="meta">[&lt;Element a at 0x1079418c8&gt;]</span></span><br><span class="line"><span class="meta">[&lt;Element span at 0x107941948&gt;]</span></span><br><span class="line"><span class="meta">[&lt;Element a at 0x1079418c8&gt;]</span></span><br><span class="line"><span class="meta">[&lt;Element li at 0x107941948&gt;, &lt;Element li at 0x107941988&gt;, &lt;Element li at 0x1079419c8&gt;, &lt;Element li at 0x107941a08&gt;]</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>第一次选择时，我们调用了<code>ancestor</code>轴，可以获取所有祖先节点。其后需要跟两个冒号，然后是节点的选择器，这里我们直接使用*，表示匹配所有节点，因此返回结果是第一个<code>li</code>节点的所有祖先节点，包括<code>html</code>、<code>body</code>、<code>div</code>和<code>ul</code>。</p>
                  <p>第二次选择时，我们又加了限定条件，这次在冒号后面加了<code>div</code>，这样得到的结果就只有<code>div</code>这个祖先节点了。</p>
                  <p>第三次选择时，我们调用了<code>attribute</code>轴，可以获取所有属性值，其后跟的选择器还是*，这代表获取节点的所有属性，返回值就是<code>li</code>节点的所有属性值。</p>
                  <p>第四次选择时，我们调用了<code>child</code>轴，可以获取所有直接子节点。这里我们又加了限定条件，选取<code>href</code>属性为<code>link1.html</code>的<code>a</code>节点。</p>
                  <p>第五次选择时，我们调用了<code>descendant</code>轴，可以获取所有子孙节点。这里我们又加了限定条件获取<code>span</code>节点，所以返回的结果只包含<code>span</code>节点而不包含<code>a</code>节点。</p>
                  <p>第六次选择时，我们调用了<code>following</code>轴，可以获取当前节点之后的所有节点。这里我们虽然使用的是*匹配，但又加了索引选择，所以只获取了第二个后续节点。</p>
                  <p>第七次选择时，我们调用了<code>following-sibling</code>轴，可以获取当前节点之后的所有同级节点。这里我们使用*匹配，所以获取了所有后续同级节点。</p>
                  <p>以上是XPath轴的简单用法，更多轴的用法可以参考：<a href="http://www.w3school.com.cn/xpath/xpath_axes.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/xpath/xpath_axes.asp</a>。</p>
                  <h2 id="15-结语"><a href="#15-结语" class="headerlink" title="15. 结语"></a>15. 结语</h2>
                  <p>到现在为止，我们基本上把可能用到的XPath选择器介绍完了。XPath功能非常强大，内置函数非常多，熟练使用之后，可以大大提升HTML信息的提取效率。</p>
                  <p>如果想查询更多XPath的用法，可以查看：<a href="http://www.w3school.com.cn/xpath/index.asp" target="_blank" rel="noopener">http://www.w3school.com.cn/xpath/index.asp</a>。</p>
                  <p>如果想查询更多Python lxml库的用法，可以查看<a href="http://lxml.de/" target="_blank" rel="noopener">http://lxml.de/</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-28 16:03:48" itemprop="dateCreated datePublished" datetime="2018-01-28T16:03:48+08:00">2018-01-28</time>
                </span>
                <span id="/5545.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 4.1-使用XPath" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>13k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>12 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5542.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5542.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 4-解析库的使用</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>上一章中，我们实现了一个最基本的爬虫，但提取页面信息时使用的是正则表达式，这还是比较烦琐，而且万一有地方写错了，可能导致匹配失败，所以使用正则表达式提取页面信息多多少少还是有些不方便。</p>
                  <p>对于网页的节点来说，它可以定义<code>id</code>、<code>class</code>或其他属性。而且节点之间还有层次关系，在网页中可以通过XPath或CSS选择器来定位一个或多个节点。那么，在页面解析时，利用XPath或CSS选择器来提取某个节点，然后再调用相应方法获取它的正文内容或者属性，不就可以提取我们想要的任意信息了吗？</p>
                  <p>在Python中，怎样实现这个操作呢？不用担心，这种解析库已经非常多，其中比较强大的库有lxml、Beautiful Soup、pyquery等，本章就来介绍这3个解析库的用法。有了它们，我们就不用再为正则表达式发愁，而且解析效率也会大大提高。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-28 15:58:40" itemprop="dateCreated datePublished" datetime="2018-01-28T15:58:40+08:00">2018-01-28</time>
                </span>
                <span id="/5542.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 4-解析库的使用" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>358</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5534.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5534.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 3.4-抓取猫眼电影排行</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节中，我们利用requests库和正则表达式来抓取猫眼电影TOP100的相关内容。requests比urllib使用更加方便，而且目前我们还没有系统学习HTML解析库，所以这里就选用正则表达式来作为解析工具。</p>
                  <h2 id="1-本节目标"><a href="#1-本节目标" class="headerlink" title="1. 本节目标"></a>1. 本节目标</h2>
                  <p>本节中，我们要提取出猫眼电影TOP100的电影名称、时间、评分、图片等信息，提取的站点URL为<a href="http://maoyan.com/board/4" target="_blank" rel="noopener">http://maoyan.com/board/4</a>，提取的结果会以文件形式保存下来。</p>
                  <h2 id="2-准备工作"><a href="#2-准备工作" class="headerlink" title="2. 准备工作"></a>2. 准备工作</h2>
                  <p>在本节开始之前，请确保已经正确安装好了requests库。如果没有安装，可以参考第1章的安装说明。</p>
                  <h2 id="3-抓取分析"><a href="#3-抓取分析" class="headerlink" title="3. 抓取分析"></a>3. 抓取分析</h2>
                  <p>我们需要抓取的目标站点为<a href="http://maoyan.com/board/4" target="_blank" rel="noopener">http://maoyan.com/board/4</a>，打开之后便可以查看到榜单信息，如图3-11所示。 <img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-11.jpg" alt="">图3-11 榜单信息</p>
                  <p>排名第一的电影是霸王别姬，页面中显示的有效信息有影片名称、主演、上映时间、上映地区、评分、图片等信息。</p>
                  <p>将网页滚动到最下方，可以发现有分页的列表，直接点击第2页，观察页面的URL和内容发生了怎样的变化，如图3-12所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-12.jpg" alt="">图3-12 页面URL变化</p>
                  <p>可以发现页面的URL变成<a href="http://maoyan.com/board/4?offset=10" target="_blank" rel="noopener">http://maoyan.com/board/4?offset=10</a>，比之前的URL多了一个参数，那就是<code>offset=10</code>，而目前显示的结果是排行11~20名的电影，初步推断这是一个偏移量的参数。再点击下一页，发现页面的URL变成了<a href="http://maoyan.com/board/4?offset=20" target="_blank" rel="noopener">http://maoyan.com/board/4?offset=20</a>，参数<code>offset</code>变成了20，而显示的结果是排行21~30的电影。</p>
                  <p>由此可以总结出规律，<code>offset</code>代表偏移量值，如果偏移量为<code>n</code>，则显示的电影序号就是<code>n+1</code>到<code>n+10</code>，每页显示10个。所以，如果想获取TOP100电影，只需要分开请求10次，而10次的<code>offset</code>参数分别设置为0、10、20、…90即可，这样获取不同的页面之后，再用正则表达式提取出相关信息，就可以得到TOP100的所有电影信息了。</p>
                  <h2 id="4-抓取首页"><a href="#4-抓取首页" class="headerlink" title="4. 抓取首页"></a>4. 抓取首页</h2>
                  <p>接下来用代码实现这个过程。首先抓取第一页的内容。我们实现了<code>get_one_page()</code>方法，并给它传入<code>url</code>参数。然后将抓取的页面结果返回，再通过<code>main()</code>方法调用。初步代码实现如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_one_page</span><span class="params">(url)</span>:</span></span><br><span class="line">    response = requests.get(url)</span><br><span class="line">    <span class="keyword">if</span> response.status_code == <span class="number">200</span>:</span><br><span class="line">        <span class="keyword">return</span> response.text</span><br><span class="line">    <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">main</span><span class="params">()</span>:</span></span><br><span class="line">    url = <span class="string">'http://maoyan.com/board/4'</span></span><br><span class="line">    html = get_one_page(url)</span><br><span class="line">    print(html)</span><br><span class="line"></span><br><span class="line">main()</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样运行之后，就可以成功获取首页的源代码了。获取源代码后，就需要解析页面，提取出我们想要的信息。</p>
                  <h2 id="5-正则提取"><a href="#5-正则提取" class="headerlink" title="5. 正则提取"></a>5. 正则提取</h2>
                  <p>接下来，回到网页看一下页面的真实源码。在开发者模式下的Network监听组件中查看源代码，如图3-13所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-13.jpg" alt="">图3-13 源代码</p>
                  <p>注意，这里不要在Elements选项卡中直接查看源码，因为那里的源码可能经过JavaScript操作而与原始请求不同，而是需要从Network选项卡部分查看原始请求得到的源码。</p>
                  <p>查看其中一个条目的源代码，如图3-14所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-14.jpg" alt="">图3-14 源代码</p>
                  <p>可以看到，一部电影信息对应的源代码是一个<code>dd</code>节点，我们用正则表达式来提取这里面的一些电影信息。首先，需要提取它的排名信息。而它的排名信息是在<code>class</code>为<code>board-index</code>的<code>i</code>节点内，这里利用非贪婪匹配来提取<code>i</code>节点内的信息，正则表达式写为：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">dd</span>&gt;</span>.*?board-index.*?&gt;(.*?)<span class="tag">&lt;/<span class="name">i</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>随后需要提取电影的图片。可以看到，后面有<code>a</code>节点，其内部有两个<code>img</code>节点。经过检查后发现，第二个<code>img</code>节点的<code>data-src</code>属性是图片的链接。这里提取第二个<code>img</code>节点的<code>data-src</code>属性，正则表达式可以改写如下：</p>
                  <figure class="highlight dts">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="params">&lt;dd&gt;</span>.*?board-index.*?&gt;(.*?)<span class="params">&lt;/i&gt;</span>.*?data-src=<span class="string">"(.*?)"</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再往后，需要提取电影的名称，它在后面的<code>p</code>节点内，<code>class</code>为<code>name</code>。所以，可以用<code>name</code>做一个标志位，然后进一步提取到其内<code>a</code>节点的正文内容，此时正则表达式改写如下：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;dd&gt;.*?board-index.*<span class="meta">?&gt;</span>(.*?)&lt;/i&gt;.*?data-src=<span class="string">"(.*?)"</span>.*?name.*?<span class="keyword">a</span>.*<span class="meta">?&gt;</span>(.*?)&lt;/<span class="keyword">a</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再提取主演、发布时间、评分等内容时，都是同样的原理。最后，正则表达式写为：</p>
                  <figure class="highlight asciidoc">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;dd&gt;.<span class="strong">*?board-index.*</span>?&gt;(.<span class="strong">*?)&lt;/i&gt;.*</span>?data-src="(.<span class="strong">*?)".*</span>?name.<span class="strong">*?a.*</span>?&gt;(.<span class="strong">*?)&lt;/a&gt;.*</span>?star.<span class="strong">*?&gt;(.*</span>?)&lt;/p&gt;.<span class="strong">*?releasetime.*</span>?&gt;(.<span class="strong">*?)&lt;/p&gt;.*</span>?integer.<span class="strong">*?&gt;(.*</span>?)&lt;/i&gt;.<span class="strong">*?fraction.*</span>?&gt;(.<span class="strong">*?)&lt;/i&gt;.*</span>?&lt;/dd&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样一个正则表达式可以匹配一个电影的结果，里面匹配了7个信息。接下来，通过调用<code>findall()</code>方法提取出所有的内容。</p>
                  <p>接下来，我们再定义解析页面的方法<code>parse_one_page()</code>，主要是通过正则表达式来从结果中提取出我们想要的内容，实现代码如下：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_one_page(html):</span><br><span class="line">    pattern = re.compile(</span><br><span class="line">        <span class="comment">'<span class="doctag">&lt;dd&gt;</span>.*?board-index.*?&gt;(.*?)<span class="doctag">&lt;/i&gt;</span>.*?data-src="(.*?)".*?name.*?a.*?&gt;(.*?)<span class="doctag">&lt;/a&gt;</span>.*?star.*?&gt;(.*?)<span class="doctag">&lt;/p&gt;</span>.*?releasetime.*?&gt;(.*?)<span class="doctag">&lt;/p&gt;</span>.*?integer.*?&gt;(.*?)<span class="doctag">&lt;/i&gt;</span>.*?fraction.*?&gt;(.*?)<span class="doctag">&lt;/i&gt;</span>.*?<span class="doctag">&lt;/dd&gt;</span>',</span></span><br><span class="line">        re.S)</span><br><span class="line">    items = re.findall(pattern, html)</span><br><span class="line">    print(items)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以成功地将一页的10个电影信息都提取出来，这是一个列表形式，输出结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="symbol">'1</span>', <span class="symbol">'http://p1.meituan.net/movie/20803f59291c47e1e116c11963ce019e68711.jpg@160w_220h_1e_1c</span>', <span class="symbol">'霸王别姬</span>', '\n                主演：张国荣,张丰毅,巩俐\n        ', <span class="symbol">'上映时间：1993-01-01</span>(<span class="name">中国香港</span>)', <span class="symbol">'9.</span>', <span class="symbol">'6</span>'), (<span class="symbol">'2</span>', <span class="symbol">'http://p0.meituan.net/movie/__40191813__4767047.jpg@160w_220h_1e_1c</span>', <span class="symbol">'肖申克的救赎</span>', '\n                主演：蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿\n        ', <span class="symbol">'上映时间：1994-10-14</span>(<span class="name">美国</span>)', <span class="symbol">'9.</span>', <span class="symbol">'5</span>'), (<span class="symbol">'3</span>', <span class="symbol">'http://p0.meituan.net/movie/fc9d78dd2ce84d20e53b6d1ae2eea4fb1515304.jpg@160w_220h_1e_1c</span>', <span class="symbol">'这个杀手不太冷</span>', '\n                主演：让·雷诺,加里·奥德曼,娜塔莉·波特曼\n        ', <span class="symbol">'上映时间：1994-09-14</span>(<span class="name">法国</span>)', <span class="symbol">'9.</span>', <span class="symbol">'5</span>'), (<span class="symbol">'4</span>', <span class="symbol">'http://p0.meituan.net/movie/23/6009725.jpg@160w_220h_1e_1c</span>', <span class="symbol">'罗马假日</span>', '\n                主演：格利高利·派克,奥黛丽·赫本,埃迪·艾伯特\n        ', <span class="symbol">'上映时间：1953-09-02</span>(<span class="name">美国</span>)', <span class="symbol">'9.</span>', <span class="symbol">'1</span>'), (<span class="symbol">'5</span>', <span class="symbol">'http://p0.meituan.net/movie/53/1541925.jpg@160w_220h_1e_1c</span>', <span class="symbol">'阿甘正传</span>', '\n                主演：汤姆·汉克斯,罗宾·怀特,加里·西尼斯\n        ', <span class="symbol">'上映时间：1994-07-06</span>(<span class="name">美国</span>)', <span class="symbol">'9.</span>', <span class="symbol">'4</span>'), (<span class="symbol">'6</span>', <span class="symbol">'http://p0.meituan.net/movie/11/324629.jpg@160w_220h_1e_1c</span>', <span class="symbol">'泰坦尼克号</span>', '\n                主演：莱昂纳多·迪卡普里奥,凯特·温丝莱特,比利·赞恩\n        ', <span class="symbol">'上映时间：1998-04-03</span>', <span class="symbol">'9.</span>', <span class="symbol">'5</span>'), (<span class="symbol">'7</span>', <span class="symbol">'http://p0.meituan.net/movie/99/678407.jpg@160w_220h_1e_1c</span>', <span class="symbol">'龙猫</span>', '\n                主演：日高法子,坂本千夏,糸井重里\n        ', <span class="symbol">'上映时间：1988-04-16</span>(<span class="name">日本</span>)', <span class="symbol">'9.</span>', <span class="symbol">'2</span>'), (<span class="symbol">'8</span>', <span class="symbol">'http://p0.meituan.net/movie/92/8212889.jpg@160w_220h_1e_1c</span>', <span class="symbol">'教父</span>', '\n                主演：马龙·白兰度,阿尔·帕西诺,詹姆斯·凯恩\n        ', <span class="symbol">'上映时间：1972-03-24</span>(<span class="name">美国</span>)', <span class="symbol">'9.</span>', <span class="symbol">'3</span>'), (<span class="symbol">'9</span>', <span class="symbol">'http://p0.meituan.net/movie/62/109878.jpg@160w_220h_1e_1c</span>', <span class="symbol">'唐伯虎点秋香</span>', '\n                主演：周星驰,巩俐,郑佩佩\n        ', <span class="symbol">'上映时间：1993-07-01</span>(<span class="name">中国香港</span>)', <span class="symbol">'9.</span>', <span class="symbol">'2</span>'), (<span class="symbol">'10</span>', <span class="symbol">'http://p0.meituan.net/movie/9bf7d7b81001a9cf8adbac5a7cf7d766132425.jpg@160w_220h_1e_1c</span>', <span class="symbol">'千与千寻</span>', '\n                主演：柊瑠美,入野自由,夏木真理\n        ', <span class="symbol">'上映时间：2001-07-20</span>(<span class="name">日本</span>)', <span class="symbol">'9.</span>', <span class="symbol">'3</span>')]</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但这样还不够，数据比较杂乱，我们再将匹配结果处理一下，遍历提取结果并生成字典，此时方法改写如下：</p>
                  <figure class="highlight xquery">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">def parse_one_page(html):</span><br><span class="line">    pattern = re.compile(</span><br><span class="line">        <span class="string">'&lt;dd&gt;.*?board-index.*?&gt;(.*?)&lt;/i&gt;.*?data-src="(.*?)".*?name.*?a.*?&gt;(.*?)&lt;/a&gt;.*?star.*?&gt;(.*?)&lt;/p&gt;.*?releasetime.*?&gt;(.*?)&lt;/p&gt;.*?integer.*?&gt;(.*?)&lt;/i&gt;.*?fraction.*?&gt;(.*?)&lt;/i&gt;.*?&lt;/dd&gt;'</span>,</span><br><span class="line">        re.S)</span><br><span class="line">    items = re.findall(pattern, html)</span><br><span class="line">    <span class="keyword">for</span> <span class="type">item</span> <span class="keyword">in</span> items:</span><br><span class="line">        yield &#123;</span><br><span class="line">            <span class="string">'index'</span>: <span class="type">item</span>[<span class="number">0</span>],</span><br><span class="line">            <span class="string">'image'</span>: <span class="type">item</span>[<span class="number">1</span>],</span><br><span class="line">            <span class="string">'title'</span>: <span class="type">item</span>[<span class="number">2</span>].<span class="keyword">strip</span>(),</span><br><span class="line">            <span class="string">'actor'</span>: <span class="type">item</span>[<span class="number">3</span>].<span class="keyword">strip</span>()[<span class="number">3</span>:] <span class="keyword">if</span> len(<span class="type">item</span>[<span class="number">3</span>]) &gt; <span class="number">3</span> <span class="keyword">else</span> <span class="string">''</span>,</span><br><span class="line">            <span class="string">'time'</span>: <span class="type">item</span>[<span class="number">4</span>].<span class="keyword">strip</span>()[<span class="number">5</span>:] <span class="keyword">if</span> len(<span class="type">item</span>[<span class="number">4</span>]) &gt; <span class="number">5</span> <span class="keyword">else</span> <span class="string">''</span>,</span><br><span class="line">            <span class="string">'score'</span>: <span class="type">item</span>[<span class="number">5</span>].<span class="keyword">strip</span>() + <span class="type">item</span>[<span class="number">6</span>].<span class="keyword">strip</span>()</span><br><span class="line">        &#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就可以成功提取出电影的排名、图片、标题、演员、时间、评分等内容了，并把它赋值为一个个的字典，形成结构化数据。运行结果如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;'image': 'http://p1.meituan.net/movie/<span class="number">2080</span>3f<span class="number">5929</span>1c47e1e116c<span class="number">1196</span>3ce019e<span class="number">6871</span>1.jpg@160w_220h_1e_1c', 'actor': '张国荣,张丰毅,巩俐', 'score': '9.6', 'index': '1', 'title': '霸王别姬', 'time': '<span class="number">1993-01-01</span>(中国香港)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/__<span class="number">40191813</span>__<span class="number">476704</span>7.jpg@160w_220h_1e_1c', 'actor': '蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿', 'score': '9.5', 'index': '2', 'title': '肖申克的救赎', 'time': '<span class="number">1994-10-14</span>(美国)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/fc9d78dd2ce84d20e53b6d1ae2eea4fb<span class="number">151530</span>4.jpg@160w_220h_1e_1c', 'actor': '让·雷诺,加里·奥德曼,娜塔莉·波特曼', 'score': '9.5', 'index': '3', 'title': '这个杀手不太冷', 'time': '<span class="number">1994-09-14</span>(法国)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/23/<span class="number">600972</span>5.jpg@160w_220h_1e_1c', 'actor': '格利高利·派克,奥黛丽·赫本,埃迪·艾伯特', 'score': '9.1', 'index': '4', 'title': '罗马假日', 'time': '<span class="number">1953-09-02</span>(美国)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/53/<span class="number">154192</span>5.jpg@160w_220h_1e_1c', 'actor': '汤姆·汉克斯,罗宾·怀特,加里·西尼斯', 'score': '9.4', 'index': '5', 'title': '阿甘正传', 'time': '<span class="number">1994-07-06</span>(美国)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/11/<span class="number">324629</span>.jpg@160w_220h_1e_1c', 'actor': '莱昂纳多·迪卡普里奥,凯特·温丝莱特,比利·赞恩', 'score': '9.5', 'index': '6', 'title': '泰坦尼克号', 'time': '<span class="number">1998-04-03</span>'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/99/<span class="number">678407</span>.jpg@160w_220h_1e_1c', 'actor': '日高法子,坂本千夏,糸井重里', 'score': '9.2', 'index': '7', 'title': '龙猫', 'time': '<span class="number">1988-04-16</span>(日本)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/92/<span class="number">821288</span>9.jpg@160w_220h_1e_1c', 'actor': '马龙·白兰度,阿尔·帕西诺,詹姆斯·凯恩', 'score': '9.3', 'index': '8', 'title': '教父', 'time': '<span class="number">1972-03-24</span>(美国)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/62/<span class="number">109878</span>.jpg@160w_220h_1e_1c', 'actor': '周星驰,巩俐,郑佩佩', 'score': '9.2', 'index': '9', 'title': '唐伯虎点秋香', 'time': '<span class="number">1993-07-01</span>(中国香港)'&#125;</span><br><span class="line">&#123;'image': 'http://p0.meituan.net/movie/9bf7d7b<span class="number">8100</span>1a9cf8adbac5a7cf7d<span class="number">76613242</span>5.jpg@160w_220h_1e_1c', 'actor': '柊瑠美,入野自由,夏木真理', 'score': '9.3', 'index': '10', 'title': '千与千寻', 'time': '<span class="number">2001-07-20</span>(日本)'&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>到此为止，我们就成功提取了单页的电影信息。</p>
                  <h2 id="6-写入文件"><a href="#6-写入文件" class="headerlink" title="6. 写入文件"></a>6. 写入文件</h2>
                  <p>随后，我们将提取的结果写入文件，这里直接写入到一个文本文件中。这里通过JSON库的<code>dumps()</code>方法实现字典的序列化，并指定<code>ensure_ascii</code>参数为<code>False</code>，这样可以保证输出结果是中文形式而不是Unicode编码。代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">write_to_json</span><span class="params">(content)</span>:</span></span><br><span class="line">    <span class="keyword">with</span> open(<span class="string">'result.txt'</span>, <span class="string">'a'</span>) <span class="keyword">as</span> f:</span><br><span class="line">        print(type(json.dumps(content)))</span><br><span class="line">        f.write(json.dumps(content, ensure_ascii=<span class="literal">False</span>,).encode(<span class="string">'utf-8'</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过调用<code>write_to_json()</code>方法即可实现将字典写入到文本文件的过程，此处的<code>content</code>参数就是一部电影的提取结果，是一个字典。</p>
                  <h2 id="7-整合代码"><a href="#7-整合代码" class="headerlink" title="7. 整合代码"></a>7. 整合代码</h2>
                  <p>最后，实现<code>main()</code>方法来调用前面实现的方法，将单页的电影结果写入到文件。相关代码如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">def</span> <span class="function"><span class="title">main</span>():</span></span><br><span class="line"><span class="function">    <span class="variable">url</span> = <span class="string">'http://maoyan.com/board/4'</span></span></span><br><span class="line"><span class="function">    <span class="variable">html</span> = <span class="title">get_one_page</span>(<span class="variable">url</span>)</span></span><br><span class="line">    <span class="variable">for</span> <span class="variable">item</span> <span class="variable"><span class="keyword">in</span></span> <span class="function"><span class="title">parse_one_page</span>(<span class="variable">html</span>):</span></span><br><span class="line"><span class="function">        <span class="title">write_to_json</span>(<span class="variable">item</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>到此为止，我们就完成了单页电影的提取，也就是首页的10部电影可以成功提取并保存到文本文件中了。</p>
                  <h2 id="8-分页爬取"><a href="#8-分页爬取" class="headerlink" title="8. 分页爬取"></a>8. 分页爬取</h2>
                  <p>因为我们需要抓取的是TOP100的电影，所以还需要遍历一下，给这个链接传入<code>offset</code>参数，实现其他90部电影的爬取，此时添加如下调用即可：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable"><span class="keyword">if</span></span> <span class="variable">__name__</span> == <span class="string">'__main__'</span>:</span><br><span class="line">    <span class="variable">for</span> <span class="variable">i</span> <span class="variable"><span class="keyword">in</span></span> <span class="function"><span class="title">range</span>(<span class="number">10</span>):</span></span><br><span class="line"><span class="function">        <span class="title">main</span>(<span class="variable">offset</span>=<span class="variable">i</span> * <span class="number">10</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里还需要将<code>main()</code>方法修改一下，接收一个<code>offset</code>值作为偏移量，然后构造URL进行爬取。实现代码如下：</p>
                  <figure class="highlight isbl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="variable">def</span> <span class="function"><span class="title">main</span>(<span class="variable">offset</span>):</span></span><br><span class="line"><span class="function">    <span class="variable">url</span> = <span class="string">'http://maoyan.com/board/4?offset='</span> + <span class="title">str</span>(<span class="variable">offset</span>)</span></span><br><span class="line">    <span class="variable">html</span> = <span class="function"><span class="title">get_one_page</span>(<span class="variable">url</span>)</span></span><br><span class="line">    <span class="variable">for</span> <span class="variable">item</span> <span class="variable"><span class="keyword">in</span></span> <span class="function"><span class="title">parse_one_page</span>(<span class="variable">html</span>):</span></span><br><span class="line"><span class="function">        <span class="title">print</span>(<span class="variable">item</span>)</span></span><br><span class="line">        <span class="function"><span class="title">write_to_file</span>(<span class="variable">item</span>)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>到此为止，我们的猫眼电影TOP100的爬虫就全部完成了，再稍微整理一下，完整的代码如下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> json</span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> requests.exceptions <span class="keyword">import</span> RequestException</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">get_one_page</span><span class="params">(url)</span>:</span></span><br><span class="line">    <span class="keyword">try</span>:</span><br><span class="line">        response = requests.get(url)</span><br><span class="line">        <span class="keyword">if</span> response.status_code == <span class="number">200</span>:</span><br><span class="line">            <span class="keyword">return</span> response.text</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line">    <span class="keyword">except</span> RequestException:</span><br><span class="line">        <span class="keyword">return</span> <span class="literal">None</span></span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">parse_one_page</span><span class="params">(html)</span>:</span></span><br><span class="line">    pattern = re.compile(<span class="string">'&lt;dd&gt;.*?board-index.*?&gt;(\d+)&lt;/i&gt;.*?data-src="(.*?)".*?name"&gt;&lt;a'</span></span><br><span class="line">                         + <span class="string">'.*?&gt;(.*?)&lt;/a&gt;.*?star"&gt;(.*?)&lt;/p&gt;.*?releasetime"&gt;(.*?)&lt;/p&gt;'</span></span><br><span class="line">                         + <span class="string">'.*?integer"&gt;(.*?)&lt;/i&gt;.*?fraction"&gt;(.*?)&lt;/i&gt;.*?&lt;/dd&gt;'</span>, re.S)</span><br><span class="line">    items = re.findall(pattern, html)</span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> items:</span><br><span class="line">        <span class="keyword">yield</span> &#123;</span><br><span class="line">            <span class="string">'index'</span>: item[<span class="number">0</span>],</span><br><span class="line">            <span class="string">'image'</span>: item[<span class="number">1</span>],</span><br><span class="line">            <span class="string">'title'</span>: item[<span class="number">2</span>],</span><br><span class="line">            <span class="string">'actor'</span>: item[<span class="number">3</span>].strip()[<span class="number">3</span>:],</span><br><span class="line">            <span class="string">'time'</span>: item[<span class="number">4</span>].strip()[<span class="number">5</span>:],</span><br><span class="line">            <span class="string">'score'</span>: item[<span class="number">5</span>] + item[<span class="number">6</span>]</span><br><span class="line">        &#125;</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">write_to_file</span><span class="params">(content)</span>:</span></span><br><span class="line">    <span class="keyword">with</span> open(<span class="string">'result.txt'</span>, <span class="string">'a'</span>, encoding=<span class="string">'utf-8'</span>) <span class="keyword">as</span> f:</span><br><span class="line">        f.write(json.dumps(content, ensure_ascii=<span class="literal">False</span>) + <span class="string">'\n'</span>)</span><br><span class="line"></span><br><span class="line"><span class="function"><span class="keyword">def</span> <span class="title">main</span><span class="params">(offset)</span>:</span></span><br><span class="line">    url = <span class="string">'http://maoyan.com/board/4?offset='</span> + str(offset)</span><br><span class="line">    html = get_one_page(url)</span><br><span class="line">    <span class="keyword">for</span> item <span class="keyword">in</span> parse_one_page(html):</span><br><span class="line">        print(item)</span><br><span class="line">        write_to_file(item)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">'__main__'</span>:</span><br><span class="line">    <span class="keyword">for</span> i <span class="keyword">in</span> range(<span class="number">10</span>):</span><br><span class="line">        main(offset=i * <span class="number">10</span>)</span><br><span class="line">        time.sleep(<span class="number">1</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>现在猫眼多了反爬虫，如果速度过快，则会无响应，所以这里又增加了一个延时等待。</p>
                  <h2 id="9-运行结果"><a href="#9-运行结果" class="headerlink" title="9. 运行结果"></a>9. 运行结果</h2>
                  <p>最后，我们运行一下代码，输出结果类似如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;'index': '1', 'image': 'http://p1.meituan.net/movie/<span class="number">2080</span>3f<span class="number">5929</span>1c47e1e116c<span class="number">1196</span>3ce019e<span class="number">6871</span>1.jpg@160w_220h_1e_1c', 'title': '霸王别姬', 'actor': '张国荣,张丰毅,巩俐', 'time': '<span class="number">1993-01-01</span>(中国香港)', 'score': '9.6'&#125;</span><br><span class="line">&#123;'index': '2', 'image': 'http://p0.meituan.net/movie/__<span class="number">40191813</span>__<span class="number">476704</span>7.jpg@160w_220h_1e_1c', 'title': '肖申克的救赎', 'actor': '蒂姆·罗宾斯,摩根·弗里曼,鲍勃·冈顿', 'time': '<span class="number">1994-10-14</span>(美国)', 'score': '9.5'&#125;</span><br><span class="line">...</span><br><span class="line">&#123;'index': '98', 'image': 'http://p0.meituan.net/movie/76/<span class="number">707338</span>9.jpg@160w_220h_1e_1c', 'title': '东京物语', 'actor': '笠智众,原节子,杉村春子', 'time': '<span class="number">1953-11-03</span>(日本)', 'score': '9.1'&#125;</span><br><span class="line">&#123;'index': '99', 'image': 'http://p0.meituan.net/movie/52/<span class="number">342029</span>3.jpg@160w_220h_1e_1c', 'title': '我爱你', 'actor': '宋在河,李彩恩,吉海延', 'time': '<span class="number">2011-02-17</span>(韩国)', 'score': '9.0'&#125;</span><br><span class="line">&#123;'index': '100', 'image': 'http://p1.meituan.net/movie/__<span class="number">44335138</span>__<span class="number">847077</span>9.jpg@160w_220h_1e_1c', 'title': '迁徙的鸟', 'actor': '雅克·贝汉,菲利普·拉波洛,Philippe Labro', 'time': '<span class="number">2001-12-12</span>(法国)', 'score': '9.1'&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里省略了中间的部分输出结果。可以看到，这样就成功地把TOP100的电影信息爬取下来了。</p>
                  <p>这时我们再看下文本文件，结果如图3-15所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-15.jpg" alt="">图3-15 运行结果</p>
                  <p>可以看到，电影信息也已全部保存到了文本文件中了，大功告成！</p>
                  <h2 id="10-本节代码"><a href="#10-本节代码" class="headerlink" title="10. 本节代码"></a>10. 本节代码</h2>
                  <p>本节的代码地址为<a href="https://github.com/Python3WebSpider/MaoYan" target="_blank" rel="noopener">https://github.com/Python3WebSpider/MaoYan</a>。</p>
                  <p>本节中，我们通过爬取猫眼TOP100的电影信息练习了requests和正则表达式的用法。这是一个最基础的实例，希望大家可以通过这个实例对爬虫的实现有一个最基本的思路，也对这两个库的用法有更深一步的了解。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-27 14:40:15" itemprop="dateCreated datePublished" datetime="2018-01-27T14:40:15+08:00">2018-01-27</time>
                </span>
                <span id="/5534.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 3.4-抓取猫眼电影排行" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>10k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5530.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5530.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 3.3-正则表达式</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>本节中，我们看一下正则表达式的相关用法。正则表达式是处理字符串的强大工具，它有自己特定的语法结构，有了它，实现字符串的检索、替换、匹配验证都不在话下。</p>
                  <p>当然，对于爬虫来说，有了它，从HTML里提取想要的信息就非常方便了。</p>
                  <h2 id="1-实例引入"><a href="#1-实例引入" class="headerlink" title="1. 实例引入"></a>1. 实例引入</h2>
                  <p>说了这么多，可能我们对它到底是个什么还是比较模糊，下面就用几个实例来看一下正则表达式的用法。</p>
                  <p>打开开源中国提供的正则表达式测试工具<a href="http://tool.oschina.net/regex/" target="_blank" rel="noopener">http://tool.oschina.net/regex/</a>，输入待匹配的文本，然后选择常用的正则表达式，就可以得出相应的匹配结果了。例如，这里输入待匹配的文本如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">Hello, my phone number <span class="keyword">is</span> <span class="number">010</span><span class="number">-86432100</span> <span class="keyword">and</span> email <span class="keyword">is</span> <span class="symbol">cqc@</span>cuiqingcai.com, <span class="keyword">and</span> my website <span class="keyword">is</span> http:<span class="comment">//cuiqingcai.com.</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这段字符串中包含了一个电话号码和一个电子邮件，接下来就尝试用正则表达式提取出来，如图3-10所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-10.jpg" alt="">图3-10 运行页面</p>
                  <p>在网页右侧选择“匹配Email地址”，就可以看到下方出现了文本中的E-mail。如果选择“匹配网址URL”，就可以看到下方出现了文本中的URL。是不是非常神奇？</p>
                  <p>其实，这里就是用了正则表达式匹配，也就是用一定的规则将特定的文本提取出来。比如，电子邮件开头是一段字符串，然后是一个@符号，最后是某个域名，这是有特定的组成格式的。另外，对于URL，开头是协议类型，然后是冒号加双斜线，最后是域名加路径。</p>
                  <p>对于URL来说，可以用下面的正则表达式匹配：</p>
                  <figure class="highlight cs">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[<span class="meta">a-zA-z</span>]+:<span class="comment">//[^\s]*</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>用这个正则表达式去匹配一个字符串，如果这个字符串中包含类似URL的文本，那就会被提取出来。</p>
                  <p>这个正则表达式看上去是乱糟糟的一团，其实不然，这里面都是有特定的语法规则的。比如，<code>a-z</code>代表匹配任意的小写字母，<code>\\s</code>表示匹配任意的空白字符，<code>*</code>就代表匹配前面的字符任意多个，这一长串的正则表达式就是这么多匹配规则的组合。</p>
                  <p>写好正则表达式后，就可以拿它去一个长字符串里匹配查找了。不论这个字符串里面有什么，只要符合我们写的规则，统统可以找出来。对于网页来说，如果想找出网页源代码里有多少URL，用匹配URL的正则表达式去匹配即可。</p>
                  <p>上面我们说了几个匹配规则，表3-2列出了常用的匹配规则。</p>
                  <p>表3-2 常用的匹配规则</p>
                  <p>模式</p>
                  <p>描述</p>
                  <p><code>\\w</code></p>
                  <p>匹配字母、数字及下划线</p>
                  <p><code>\\W</code></p>
                  <p>匹配不是字母、数字及下划线的字符</p>
                  <p><code>\\s</code></p>
                  <p>匹配任意空白字符，等价于<code>[\\t\\n\\r\\f]</code></p>
                  <p><code>\\S</code></p>
                  <p>匹配任意非空字符</p>
                  <p><code>\\d</code></p>
                  <p>匹配任意数字，等价于<code>[0-9]</code></p>
                  <p><code>\\D</code></p>
                  <p>匹配任意非数字的字符</p>
                  <p><code>\\A</code></p>
                  <p>匹配字符串开头</p>
                  <p><code>\\Z</code></p>
                  <p>匹配字符串结尾，如果存在换行，只匹配到换行前的结束字符串</p>
                  <p><code>\\z</code></p>
                  <p>匹配字符串结尾，如果存在换行，同时还会匹配换行符</p>
                  <p><code>\\G</code></p>
                  <p>匹配最后匹配完成的位置</p>
                  <p><code>\\n</code></p>
                  <p>匹配一个换行符</p>
                  <p><code>\\t</code></p>
                  <p>匹配一个制表符</p>
                  <p><code>^</code></p>
                  <p>匹配一行字符串的开头</p>
                  <p><code>$</code></p>
                  <p>匹配一行字符串的结尾</p>
                  <p><code>.</code></p>
                  <p>匹配任意字符，除了换行符，当<code>re.DOTALL</code>标记被指定时，则可以匹配包括换行符的任意字符</p>
                  <p><code>[...]</code></p>
                  <p>用来表示一组字符，单独列出，比如<code>[amk]</code>匹配<code>a</code>、<code>m</code>或<code>k</code></p>
                  <p><code>[^...]</code></p>
                  <p>不在<code>[]</code>中的字符，比如<code>[^abc]</code>匹配除了<code>a</code>、<code>b</code>、<code>c</code>之外的字符</p>
                  <p><code>*</code></p>
                  <p>匹配0个或多个表达式</p>
                  <p><code>+</code></p>
                  <p>匹配1个或多个表达式</p>
                  <p><code>?</code></p>
                  <p>匹配0个或1个前面的正则表达式定义的片段，非贪婪方式</p>
                  <p><code>{n}</code></p>
                  <p>精确匹配<code>n</code>个前面的表达式</p>
                  <p><code>{n, m}</code></p>
                  <p>匹配<code>n</code>到<code>m</code>次由前面正则表达式定义的片段，贪婪方式</p>
                  <p><code>a|b</code></p>
                  <p>匹配<code>a</code>或<code>b</code></p>
                  <p><code>( )</code></p>
                  <p>匹配括号内的表达式，也表示一个组</p>
                  <p>看完了之后，可能有点晕晕的吧，不过不用担心，后面我们会详细讲解一些常见规则的用法。</p>
                  <p>其实正则表达式不是Python独有的，它也可以用在其他编程语言中。但是Python的re库提供了整个正则表达式的实现，利用这个库，可以在Python中使用正则表达式。在Python中写正则表达式几乎都用这个库，下面就来了解它的一些常用方法。</p>
                  <h2 id="2-match"><a href="#2-match" class="headerlink" title="2. match()"></a>2. <code>match()</code></h2>
                  <p>这里首先介绍第一个常用的匹配方法——<code>match()</code>，向它传入要匹配的字符串以及正则表达式，就可以检测这个正则表达式是否匹配字符串。</p>
                  <p><code>match()</code>方法会尝试从字符串的起始位置匹配正则表达式，如果匹配，就返回匹配成功的结果；如果不匹配，就返回<code>None</code>。示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'Hello 123 4567 World_This is a Regex Demo'</span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(len(content)</span></span>)</span><br><span class="line">result = re.match(<span class="string">'^Hello\s\d\d\d\s\d&#123;4&#125;\s\w&#123;10&#125;'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.group()</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.span()</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">41</span></span><br><span class="line">&lt;_sre.SRE_Match object; span=(<span class="number">0</span>, <span class="number">25</span>), match='Hello <span class="number">123</span> <span class="number">4567</span> World_This'&gt;</span><br><span class="line">Hello <span class="number">123</span> <span class="number">4567</span> World_This</span><br><span class="line">(<span class="number">0</span>, <span class="number">25</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里首先声明了一个字符串，其中包含英文字母、空白字符、数字等。接下来，我们写一个正则表达式：</p>
                  <figure class="highlight tex">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">^Hello<span class="tag">\<span class="name">s</span></span><span class="tag">\<span class="name">d</span></span><span class="tag">\<span class="name">d</span></span><span class="tag">\<span class="name">d</span></span><span class="tag">\<span class="name">s</span></span><span class="tag">\<span class="name">d</span><span class="string">&#123;4&#125;</span></span><span class="tag">\<span class="name">s</span></span><span class="tag">\<span class="name">w</span><span class="string">&#123;10&#125;</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>用它来匹配这个长字符串。开头的<code>^</code>是匹配字符串的开头，也就是以<code>Hello</code>开头；然后<code>\\s</code>匹配空白字符，用来匹配目标字符串的空格；<code>\\d</code>匹配数字，3个<code>\\d</code>匹配123；然后再写1个<code>\\s</code>匹配空格；后面还有<code>4567</code>，我们其实可以依然用4个<code>\\d</code>来匹配，但是这么写比较烦琐，所以后面可以跟<code>{4}</code>以代表匹配前面的规则4次，也就是匹配4个数字；然后后面再紧接1个空白字符，最后<code>\\w{10}</code>匹配10个字母及下划线。我们注意到，这里其实并没有把目标字符串匹配完，不过这样依然可以进行匹配，只不过匹配结果短一点而已。</p>
                  <p>而在<code>match()</code>方法中，第一个参数传入了正则表达式，第二个参数传入了要匹配的字符串。</p>
                  <p>打印输出结果，可以看到结果是<code>SRE_Match</code>对象，这证明成功匹配。该对象有两个方法：<code>group()</code>方法可以输出匹配到的内容，结果是<code>Hello 123 4567 World_This</code>，这恰好是正则表达式规则所匹配的内容；<code>span()</code>方法可以输出匹配的范围，结果是<code>(0, 25)</code>，这就是匹配到的结果字符串在原字符串中的位置范围。</p>
                  <p>通过上面的例子，我们基本了解了如何在Python中使用正则表达式来匹配一段文字。</p>
                  <h3 id="匹配目标"><a href="#匹配目标" class="headerlink" title="匹配目标"></a>匹配目标</h3>
                  <p>刚才我们用<code>match()</code>方法可以得到匹配到的字符串内容，但是如果想从字符串中提取一部分内容，该怎么办呢？就像最前面的实例一样，从一段文本中提取出邮件或电话号码等内容。</p>
                  <p>这里可以使用<code>()</code>括号将想提取的子字符串括起来。<code>()</code>实际上标记了一个子表达式的开始和结束位置，被标记的每个子表达式会依次对应每一个分组，调用<code>group()</code>方法传入分组的索引即可获取提取的结果。示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'Hello 1234567 World_This is a Regex Demo'</span></span><br><span class="line">result = re.match(<span class="string">'^Hello\s(\d+)\sWorld'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.group()</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.group(<span class="number">1</span>)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.span()</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们想把字符串中的<code>1234567</code>提取出来，此时可以将数字部分的正则表达式用<code>()</code>括起来，然后调用了<code>group(1)</code>获取匹配结果。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;_sre.SRE_Match object; span=(<span class="number">0</span>, <span class="number">19</span>), match='Hello <span class="number">1234567</span> World'&gt;</span><br><span class="line">Hello <span class="number">1234567</span> World</span><br><span class="line"><span class="number">1234567</span></span><br><span class="line">(<span class="number">0</span>, <span class="number">19</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，我们成功得到了<code>1234567</code>。这里用的是<code>group(1)</code>，它与<code>group()</code>有所不同，后者会输出完整的匹配结果，而前者会输出第一个被<code>()</code>包围的匹配结果。假如正则表达式后面还有<code>()</code>包括的内容，那么可以依次用<code>group(2)</code>、<code>group(3)</code>等来获取。</p>
                  <h3 id="通用匹配"><a href="#通用匹配" class="headerlink" title="通用匹配"></a>通用匹配</h3>
                  <p>刚才我们写的正则表达式其实比较复杂，出现空白字符我们就写<code>\\s</code>匹配，出现数字我们就用<code>\\d</code>匹配，这样的工作量非常大。其实完全没必要这么做，因为还有一个万能匹配可以用，那就是<code>.*</code>（点星）。其中<code>.</code>（点）可以匹配任意字符（除换行符），<code>*</code>（星）代表匹配前面的字符无限次，所以它们组合在一起就可以匹配任意字符了。有了它，我们就不用挨个字符地匹配了。</p>
                  <p>接着上面的例子，我们可以改写一下正则表达式：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'Hello 123 4567 World_This is a Regex Demo'</span></span><br><span class="line">result = re.match(<span class="string">'^Hello.*Demo$'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.group()</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.span()</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们将中间部分直接省略，全部用<code>.*</code>来代替，最后加一个结尾字符串就好了。运行结果如下：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;_sre.SRE_Match object; span=(<span class="number">0</span>, <span class="number">41</span>), match='Hello <span class="number">123</span> <span class="number">4567</span> World_This is a Regex Demo'&gt;</span><br><span class="line">Hello <span class="number">123</span> <span class="number">4567</span> World_This is a Regex Demo</span><br><span class="line">(<span class="number">0</span>, <span class="number">41</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，<code>group()</code>方法输出了匹配的全部字符串，也就是说我们写的正则表达式匹配到了目标字符串的全部内容；<code>span()</code>方法输出<code>(0, 41)</code>，这是整个字符串的长度。</p>
                  <p>因此，我们可以使用<code>.*</code>简化正则表达式的书写。</p>
                  <h3 id="贪婪与非贪婪"><a href="#贪婪与非贪婪" class="headerlink" title="贪婪与非贪婪"></a>贪婪与非贪婪</h3>
                  <p>使用上面的通用匹配<code>.*</code>时，可能有时候匹配到的并不是我们想要的结果。看下面的例子：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'Hello 1234567 World_This is a Regex Demo'</span></span><br><span class="line">result = re.match(<span class="string">'^He.*(\d+).*Demo$'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.group(<span class="number">1</span>)</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们依然想获取中间的数字，所以中间依然写的是<code>(\\d+)</code>。而数字两侧由于内容比较杂乱，所以想省略来写，都写成 <code>.*</code>。最后，组成<code>^He.*(\\d+).*Demo$</code>，看样子并没有什么问题。我们看下运行结果：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;_sre.SRE_Match object; span=(<span class="number">0</span>, <span class="number">40</span>), match='Hello <span class="number">1234567</span> World_This is a Regex Demo'&gt;</span><br><span class="line"><span class="number">7</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>奇怪的事情发生了，我们只得到了7这个数字，这是怎么回事呢？</p>
                  <p>这里就涉及一个贪婪匹配与非贪婪匹配的问题了。在贪婪匹配下，<code>.*</code>会匹配尽可能多的字符。正则表达式中<code>.*</code>后面是<code>\\d+</code>，也就是至少一个数字，并没有指定具体多少个数字，因此，<code>.*</code>就尽可能匹配多的字符，这里就把<code>123456</code>匹配了，给<code>\\d+</code>留下一个可满足条件的数字7，最后得到的内容就只有数字7了。</p>
                  <p>但这很明显会给我们带来很大的不便。有时候，匹配结果会莫名其妙少了一部分内容。其实，这里只需要使用非贪婪匹配就好了。非贪婪匹配的写法是<code>.*?</code>，多了一个<code>?</code>，那么它可以达到怎样的效果？我们再用实例看一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'Hello 1234567 World_This is a Regex Demo'</span></span><br><span class="line">result = re.match(<span class="string">'^He.*?(\d+).*Demo$'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result.group(<span class="number">1</span>)</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们只是将第一个<code>.*</code>改成了<code>.*?</code>，转变为非贪婪匹配。结果如下：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;_sre.SRE_Match object; span=(<span class="number">0</span>, <span class="number">40</span>), match='Hello <span class="number">1234567</span> World_This is a Regex Demo'&gt;</span><br><span class="line"><span class="number">1234567</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此时就可以成功获取<code>1234567</code>了。原因可想而知，贪婪匹配是尽可能匹配多的字符，非贪婪匹配就是尽可能匹配少的字符。当<code>.*?</code>匹配到<code>Hello</code>后面的空白字符时，再往后的字符就是数字了，而<code>\\d+</code>恰好可以匹配，那么这里<code>.*?</code>就不再进行匹配，交给<code>\\d+</code>去匹配后面的数字。所以这样<code>.*?</code>匹配了尽可能少的字符，<code>\\d+</code>的结果就是<code>1234567</code>了。</p>
                  <p>所以说，在做匹配的时候，字符串中间尽量使用非贪婪匹配，也就是用<code>.*?</code>来代替<code>.*</code>，以免出现匹配结果缺失的情况。</p>
                  <p>但这里需要注意，如果匹配的结果在字符串结尾，<code>.*?</code>就有可能匹配不到任何内容了，因为它会匹配尽可能少的字符。例如：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'http://weibo.com/comment/kEraCN'</span></span><br><span class="line">result1 = re.match(<span class="string">'http.*?comment/(.*?)'</span>, <span class="attribute">content</span>)</span><br><span class="line">result2 = re.match(<span class="string">'http.*?comment/(.*)'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(<span class="string">'result1'</span>, result1.group(<span class="number">1</span>)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(<span class="string">'result2'</span>, result2.group(<span class="number">1</span>)</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight properties">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">result1</span> <span class="string"></span></span><br><span class="line"><span class="attr">result2</span> <span class="string">kEraCN</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以观察到，<code>.*?</code>没有匹配到任何结果，而<code>.*</code>则尽量匹配多的内容，成功得到了匹配结果。</p>
                  <h3 id="修饰符"><a href="#修饰符" class="headerlink" title="修饰符"></a>修饰符</h3>
                  <p>正则表达式可以包含一些可选标志修饰符来控制匹配的模式。修饰符被指定为一个可选的标志。我们用实例来看一下：</p>
                  <figure class="highlight python">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">content = <span class="string">'''Hello 1234567 World_This</span></span><br><span class="line"><span class="string">is a Regex Demo</span></span><br><span class="line"><span class="string">'''</span></span><br><span class="line">result = re.match(<span class="string">'^He.*?(\d+).*?Demo$'</span>, content)</span><br><span class="line">print(result.group(<span class="number">1</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>和上面的例子相仿，我们在字符串中加了换行符，正则表达式还是一样的，用来匹配其中的数字。看一下运行结果：</p>
                  <figure class="highlight pgsql">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">AttributeError Traceback (most recent <span class="keyword">call</span> last)</span><br><span class="line">&lt;ipython-<span class="keyword">input</span><span class="number">-18</span>-c7d232b39645&gt; <span class="keyword">in</span> &lt;module&gt;()</span><br><span class="line">      <span class="number">5</span> <span class="string">'''</span></span><br><span class="line"><span class="string">      6 result = re.match('</span>^He.*?(\d+).*?Demo$<span class="string">', content)</span></span><br><span class="line"><span class="string">----&gt; 7 print(result.group(1))</span></span><br><span class="line"><span class="string"></span></span><br><span class="line"><span class="string">AttributeError: '</span>NoneTyp<span class="string">e' object has no attribute '</span><span class="keyword">group</span><span class="string">'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行直接报错，也就是说正则表达式没有匹配到这个字符串，返回结果为<code>None</code>，而我们又调用了<code>group()</code>方法导致<code>AttributeError</code>。</p>
                  <p>那么，为什么加了一个换行符，就匹配不到了呢？这是因为<code>\\.</code>匹配的是除换行符之外的任意字符，当遇到换行符时，<code>.*?</code>就不能匹配了，所以导致匹配失败。这里只需加一个修饰符<code>re.S</code>，即可修正这个错误：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">result</span> = re.match(<span class="string">'^He.*?(\d+).*?Demo$'</span>, content, re.S)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个修饰符的作用是使<code>.</code>匹配包括换行符在内的所有字符。此时运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">1234567</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这个<code>re.S</code>在网页匹配中经常用到。因为HTML节点经常会有换行，加上它，就可以匹配节点与节点之间的换行了。</p>
                  <p>另外，还有一些修饰符，在必要的情况下也可以使用，如表3-3所示。</p>
                  <p>表3-3 修饰符</p>
                  <p>修饰符</p>
                  <p>描述</p>
                  <p><code>re.I</code></p>
                  <p>使匹配对大小写不敏感</p>
                  <p><code>re.L</code></p>
                  <p>做本地化识别（locale-aware）匹配</p>
                  <p><code>re.M</code></p>
                  <p>多行匹配，影响<code>^</code>和<code>$</code></p>
                  <p><code>re.S</code></p>
                  <p>使<code>.</code>匹配包括换行在内的所有字符</p>
                  <p><code>re.U</code></p>
                  <p>根据Unicode字符集解析字符。这个标志影响<code>\\w</code>、<code>\\W</code>、 <code>\\b</code>和<code>\\B</code></p>
                  <p><code>re.X</code></p>
                  <p>该标志通过给予你更灵活的格式以便你将正则表达式写得更易于理解</p>
                  <p>在网页匹配中，较为常用的有<code>re.S</code>和<code>re.I</code>。</p>
                  <h3 id="转义匹配"><a href="#转义匹配" class="headerlink" title="转义匹配"></a>转义匹配</h3>
                  <p>我们知道正则表达式定义了许多匹配模式，如<code>.</code>匹配除换行符以外的任意字符，但是如果目标字符串里面就包含<code>.</code>，那该怎么办呢？</p>
                  <p>这里就需要用到转义匹配了，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'(百度)www.baidu.com'</span></span><br><span class="line">result = re.match(<span class="string">'\(百度\)www\.baidu\.com'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当遇到用于正则匹配模式的特殊字符时，在前面加反斜线转义一下即可。例如<code>.</code>就可以用<code>\\.</code>来匹配，运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">_sre.SRE_Match</span> <span class="attr">object</span>; <span class="attr">span</span>=<span class="string">(0,</span> <span class="attr">17</span>), <span class="attr">match</span>=<span class="string">'(百度)www.baidu.com'</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这里成功匹配到了原字符串。</p>
                  <p>这些是写正则表达式常用的几个知识点，熟练掌握它们对后面写正则表达式匹配非常有帮助。</p>
                  <h2 id="3-search"><a href="#3-search" class="headerlink" title="3. search()"></a>3. <code>search()</code></h2>
                  <p>前面提到过，<code>match()</code>方法是从字符串的开头开始匹配的，一旦开头不匹配，那么整个匹配就失败了。我们看下面的例子：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line"><span class="attribute">content</span> = <span class="string">'Extra stings Hello 1234567 World_This is a Regex Demo Extra stings'</span></span><br><span class="line">result = re.match(<span class="string">'Hello.*?(\d+).*?Demo'</span>, <span class="attribute">content</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(result)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里的字符串以<code>Extra</code>开头，但是正则表达式以<code>Hello</code>开头，整个正则表达式是字符串的一部分，但是这样匹配是失败的。运行结果如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">None</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因为<code>match()</code>方法在使用时需要考虑到开头的内容，这在做匹配时并不方便。它更适合用来检测某个字符串是否符合某个正则表达式的规则。</p>
                  <p>这里就有另外一个方法<code>search()</code>，它在匹配时会扫描整个字符串，然后返回第一个成功匹配的结果。也就是说，正则表达式可以是字符串的一部分，在匹配时，<code>search()</code>方法会依次扫描字符串，直到找到第一个符合规则的字符串，然后返回匹配内容，如果搜索完了还没有找到，就返回<code>None</code>。</p>
                  <p>我们把上面代码中的<code>match()</code>方法修改成<code>search()</code>，再看下运行结果：</p>
                  <figure class="highlight lsl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;_sre.SRE_Match object; span=(<span class="number">13</span>, <span class="number">53</span>), match='Hello <span class="number">1234567</span> World_This is a Regex Demo'&gt;</span><br><span class="line"><span class="number">1234567</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这时就得到了匹配结果。</p>
                  <p>因此，为了匹配方便，我们可以尽量使用<code>search()</code>方法。</p>
                  <p>下面再用几个实例来看看<code>search()</code>方法的用法。</p>
                  <p>首先，这里有一段待匹配的HTML文本，接下来写几个正则表达式实例来实现相应信息的提取：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = '''<span class="tag">&lt;<span class="name">div</span> <span class="attr">id</span>=<span class="string">"songs-list"</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">h2</span> <span class="attr">class</span>=<span class="string">"title"</span>&gt;</span>经典老歌<span class="tag">&lt;/<span class="name">h2</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">p</span> <span class="attr">class</span>=<span class="string">"introduction"</span>&gt;</span></span><br><span class="line">        经典老歌列表</span><br><span class="line">    <span class="tag">&lt;/<span class="name">p</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;<span class="name">ul</span> <span class="attr">id</span>=<span class="string">"list"</span> <span class="attr">class</span>=<span class="string">"list-group"</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">li</span> <span class="attr">data-view</span>=<span class="string">"2"</span>&gt;</span>一路上有你<span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">li</span> <span class="attr">data-view</span>=<span class="string">"7"</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"/2.mp3"</span> <span class="attr">singer</span>=<span class="string">"任贤齐"</span>&gt;</span>沧海一声笑<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">li</span> <span class="attr">data-view</span>=<span class="string">"4"</span> <span class="attr">class</span>=<span class="string">"active"</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"/3.mp3"</span> <span class="attr">singer</span>=<span class="string">"齐秦"</span>&gt;</span>往事随风<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">li</span> <span class="attr">data-view</span>=<span class="string">"6"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"/4.mp3"</span> <span class="attr">singer</span>=<span class="string">"beyond"</span>&gt;</span>光辉岁月<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">li</span> <span class="attr">data-view</span>=<span class="string">"5"</span>&gt;</span><span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"/5.mp3"</span> <span class="attr">singer</span>=<span class="string">"陈慧琳"</span>&gt;</span>记事本<span class="tag">&lt;/<span class="name">a</span>&gt;</span><span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;<span class="name">li</span> <span class="attr">data-view</span>=<span class="string">"5"</span>&gt;</span></span><br><span class="line">            <span class="tag">&lt;<span class="name">a</span> <span class="attr">href</span>=<span class="string">"/6.mp3"</span> <span class="attr">singer</span>=<span class="string">"邓丽君"</span>&gt;</span><span class="tag">&lt;<span class="name">i</span> <span class="attr">class</span>=<span class="string">"fa fa-user"</span>&gt;</span><span class="tag">&lt;/<span class="name">i</span>&gt;</span>但愿人长久<span class="tag">&lt;/<span class="name">a</span>&gt;</span></span><br><span class="line">        <span class="tag">&lt;/<span class="name">li</span>&gt;</span></span><br><span class="line">    <span class="tag">&lt;/<span class="name">ul</span>&gt;</span></span><br><span class="line"><span class="tag">&lt;/<span class="name">div</span>&gt;</span>'''</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以观察到，<code>ul</code>节点里有许多<code>li</code>节点，其中<code>li</code>节点中有的包含<code>a</code>节点，有的不包含<code>a</code>节点，<code>a</code>节点还有一些相应的属性——超链接和歌手名。</p>
                  <p>首先，我们尝试提取<code>class</code>为<code>active</code>的<code>li</code>节点内部的超链接包含的歌手名和歌名，此时需要提取第三个<code>li</code>节点下<code>a</code>节点的<code>singer</code>属性和文本。</p>
                  <p>此时正则表达式可以以<code>li</code>开头，然后寻找一个标志符<code>active</code>，中间的部分可以用<code>.*?</code>来匹配。接下来，要提取<code>singer</code>这个属性值，所以还需要写入<code>singer=&quot;(.*?)&quot;</code>，这里需要提取的部分用小括号括起来，以便用<code>group()</code>方法提取出来，它的两侧边界是双引号。然后还需要匹配<code>a</code>节点的文本，其中它的左边界是<code>\&gt;</code>，右边界是<code>&lt;/a&gt;</code>。然后目标内容依然用<code>(.*?)</code>来匹配，所以最后的正则表达式就变成了：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;li.*?active.*?<span class="attribute">singer</span>=<span class="string">"(.*?)"</span>&gt;(.*?)&lt;/a&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后再调用<code>search()</code>方法，它会搜索整个HTML文本，找到符合正则表达式的第一个内容返回。</p>
                  <p>另外，由于代码有换行，所以这里第三个参数需要传入<code>re.S</code>。整个匹配代码如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = re.<span class="keyword">search</span>('&lt;<span class="keyword">li</span>.*?active.*?singer=<span class="string">"(.*?)"</span>&gt;(.*?)&lt;/a&gt;', html, re.S)</span><br><span class="line"><span class="keyword">if</span> result:</span><br><span class="line">    <span class="keyword">print</span>(result.<span class="built_in">group</span>(1), result.<span class="built_in">group</span>(2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>由于需要获取的歌手和歌名都已经用小括号包围，所以可以用<code>group()</code>方法获取。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight plain">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">齐秦 往事随风</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，这正是<code>class</code>为<code>active</code>的<code>li</code>节点内部的超链接包含的歌手名和歌名。</p>
                  <p>如果正则表达式不加<code>active</code>（也就是匹配不带<code>class</code>为<code>active</code>的节点内容），那会怎样呢？我们将正则表达式中的<code>active</code>去掉，代码改写如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = re.<span class="keyword">search</span>('&lt;<span class="keyword">li</span>.*?singer=<span class="string">"(.*?)"</span>&gt;(.*?)&lt;/a&gt;', html, re.S)</span><br><span class="line"><span class="keyword">if</span> result:</span><br><span class="line">    <span class="keyword">print</span>(result.<span class="built_in">group</span>(1), result.<span class="built_in">group</span>(2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>由于search()方法会返回第一个符合条件的匹配目标，这里结果就变了：</p>
                  <figure class="highlight plain">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">任贤齐 沧海一声笑</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>把<code>active</code>标签去掉后，从字符串开头开始搜索，此时符合条件的节点就变成了第二个<code>li</code>节点，后面的就不再匹配，所以运行结果就变成第二个<code>li</code>节点中的内容。</p>
                  <p>注意，在上面的两次匹配中，<code>search()</code>方法的第三个参数都加了<code>re.S</code>，这使得<code>.*?</code>可以匹配换行，所以含有换行的<code>li</code>节点被匹配到了。如果我们将其去掉，结果会是什么？代码如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">result = re.<span class="keyword">search</span>('&lt;<span class="keyword">li</span>.*?singer=<span class="string">"(.*?)"</span>&gt;(.*?)&lt;/a&gt;', html)</span><br><span class="line"><span class="keyword">if</span> result:</span><br><span class="line">    <span class="keyword">print</span>(result.<span class="built_in">group</span>(1), result.<span class="built_in">group</span>(2))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight armasm">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">beyond </span>光辉岁月</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，结果变成了第四个<code>li</code>节点的内容。这是因为第二个和第三个<code>li</code>节点都包含了换行符，去掉<code>re.S</code>之后，<code>.*?</code>已经不能匹配换行符，所以正则表达式不会匹配到第二个和第三个<code>li</code>节点，而第四个<code>li</code>节点中不包含换行符，所以成功匹配。</p>
                  <p>由于绝大部分的HTML文本都包含了换行符，所以尽量都需要加上<code>re.S</code>修饰符，以免出现匹配不到的问题。</p>
                  <h2 id="4-findall"><a href="#4-findall" class="headerlink" title="4. findall()"></a>4. <code>findall()</code></h2>
                  <p>前面我们介绍了<code>search()</code>方法的用法，它可以返回匹配正则表达式的第一个内容，但是如果想要获取匹配正则表达式的所有内容，那该怎么办呢？这时就要借助<code>findall()</code>方法了。该方法会搜索整个字符串，然后返回匹配正则表达式的所有内容。</p>
                  <p>还是上面的HTML文本，如果想获取所有<code>a</code>节点的超链接、歌手和歌名，就可以将<code>search()</code>方法换成<code>findall()</code>方法。如果有返回结果的话，就是列表类型，所以需要遍历一下来依次获取每组内容。代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">results = re.findall(<span class="string">'&lt;li.*?href="(.*?)".*?singer="(.*?)"&gt;(.*?)&lt;/a&gt;'</span>, html, re.S)</span><br><span class="line"><span class="builtin-name">print</span>(results)</span><br><span class="line"><span class="builtin-name">print</span>(type(results))</span><br><span class="line"><span class="keyword">for</span> result <span class="keyword">in</span> results:</span><br><span class="line">    <span class="builtin-name">print</span>(result)</span><br><span class="line">    <span class="builtin-name">print</span>(result[0], result[1], result[2])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">[(<span class="string">'/2.mp3'</span>, <span class="string">'任贤齐'</span>, <span class="string">'沧海一声笑'</span>), (<span class="string">'/3.mp3'</span>, <span class="string">'齐秦'</span>, <span class="string">'往事随风'</span>), (<span class="string">'/4.mp3'</span>, <span class="string">'beyond'</span>, <span class="string">'光辉岁月'</span>), (<span class="string">'/5.mp3'</span>, <span class="string">'陈慧琳'</span>, <span class="string">'记事本'</span>), (<span class="string">'/6.mp3'</span>, <span class="string">'邓丽君'</span>, <span class="string">'但愿人长久'</span>)]</span><br><span class="line">&lt;class <span class="string">'list'</span>&gt;</span><br><span class="line">(<span class="string">'/2.mp3'</span>, <span class="string">'任贤齐'</span>, <span class="string">'沧海一声笑'</span>)</span><br><span class="line">/<span class="number">2</span><span class="selector-class">.mp3</span> 任贤齐 沧海一声笑</span><br><span class="line">(<span class="string">'/3.mp3'</span>, <span class="string">'齐秦'</span>, <span class="string">'往事随风'</span>)</span><br><span class="line">/<span class="number">3</span><span class="selector-class">.mp3</span> 齐秦 往事随风</span><br><span class="line">(<span class="string">'/4.mp3'</span>, <span class="string">'beyond'</span>, <span class="string">'光辉岁月'</span>)</span><br><span class="line">/<span class="number">4</span><span class="selector-class">.mp3</span> beyond 光辉岁月</span><br><span class="line">(<span class="string">'/5.mp3'</span>, <span class="string">'陈慧琳'</span>, <span class="string">'记事本'</span>)</span><br><span class="line">/<span class="number">5</span><span class="selector-class">.mp3</span> 陈慧琳 记事本</span><br><span class="line">(<span class="string">'/6.mp3'</span>, <span class="string">'邓丽君'</span>, <span class="string">'但愿人长久'</span>)</span><br><span class="line">/<span class="number">6</span><span class="selector-class">.mp3</span> 邓丽君 但愿人长久</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，返回的列表中的每个元素都是元组类型，我们用对应的索引依次取出即可。</p>
                  <p>如果只是获取第一个内容，可以用<code>search()</code>方法。当需要提取多个内容时，可以用<code>findall()</code>方法。</p>
                  <h2 id="5-sub"><a href="#5-sub" class="headerlink" title="5. sub()"></a>5. <code>sub()</code></h2>
                  <p>除了使用正则表达式提取信息外，有时候还需要借助它来修改文本。比如，想要把一串文本中的所有数字都去掉，如果只用字符串的<code>replace()</code>方法，那就太烦琐了，这时可以借助<code>sub()</code>方法。示例如下：</p>
                  <figure class="highlight coffeescript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">content = <span class="string">'54aK54yr5oiR54ix5L2g'</span></span><br><span class="line">content = re.sub(<span class="string">'\d+'</span>, <span class="string">''</span>, content)</span><br><span class="line"><span class="built_in">print</span>(content)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ebnf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">aKyroiRixLg</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里只需要给第一个参数传入<code>\\d+</code>来匹配所有的数字，第二个参数为替换成的字符串（如果去掉该参数的话，可以赋值为空），第三个参数是原字符串。</p>
                  <p>在上面的HTML文本中，如果想获取所有<code>li</code>节点的歌名，直接用正则表达式来提取可能比较烦琐。比如，可以写成这样子：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">results = re.findall(<span class="comment">'<span class="doctag">&lt;li.*?&gt;</span>\s*?(<span class="doctag">&lt;a.*?&gt;</span>)?(\w+)(<span class="doctag">&lt;/a&gt;</span>)?\s*?<span class="doctag">&lt;/li&gt;</span>', html, re.S)</span></span><br><span class="line"><span class="keyword">for</span> result <span class="keyword">in</span> results:</span><br><span class="line">    print(result[<span class="number">1</span>])</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight plain">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">一路上有你</span><br><span class="line">沧海一声笑</span><br><span class="line">往事随风</span><br><span class="line">光辉岁月</span><br><span class="line">记事本</span><br><span class="line">但愿人长久</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此时借助<code>sub()</code>方法就比较简单了。可以先用<code>sub()</code>方法将<code>a</code>节点去掉，只留下文本，然后再利用<code>findall()</code>提取就好了：</p>
                  <figure class="highlight vbnet">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">html = re.<span class="keyword">sub</span>(<span class="comment">'<span class="doctag">&lt;a.*?&gt;</span>|<span class="doctag">&lt;/a&gt;</span>', '', html)</span></span><br><span class="line">print(html)</span><br><span class="line">results = re.findall(<span class="comment">'<span class="doctag">&lt;li.*?&gt;</span>(.*?)<span class="doctag">&lt;/li&gt;</span>', html, re.S)</span></span><br><span class="line"><span class="keyword">for</span> result <span class="keyword">in</span> results:</span><br><span class="line">    print(result.strip())</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight stata">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;div id=<span class="string">"songs-list"</span>&gt;</span><br><span class="line">    &lt;h2 <span class="keyword">class</span>=<span class="string">"title"</span>&gt;经典老歌&lt;/h2&gt;</span><br><span class="line">    &lt;p <span class="keyword">class</span>=<span class="string">"introduction"</span>&gt;</span><br><span class="line">        经典老歌列表</span><br><span class="line">    &lt;/p&gt;</span><br><span class="line">    &lt;ul id=<span class="string">"list"</span> <span class="keyword">class</span>=<span class="string">"list-group"</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">li</span> data-<span class="keyword">view</span>=<span class="string">"2"</span>&gt;一路上有你&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">li</span> data-<span class="keyword">view</span>=<span class="string">"7"</span>&gt;</span><br><span class="line">            沧海一声笑</span><br><span class="line">        &lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">li</span> data-<span class="keyword">view</span>=<span class="string">"4"</span> <span class="keyword">class</span>=<span class="string">"active"</span>&gt;</span><br><span class="line">            往事随风</span><br><span class="line">        &lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">li</span> data-<span class="keyword">view</span>=<span class="string">"6"</span>&gt;光辉岁月&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">li</span> data-<span class="keyword">view</span>=<span class="string">"5"</span>&gt;记事本&lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">        &lt;<span class="keyword">li</span> data-<span class="keyword">view</span>=<span class="string">"5"</span>&gt;</span><br><span class="line">            但愿人长久</span><br><span class="line">        &lt;/<span class="keyword">li</span>&gt;</span><br><span class="line">    &lt;/ul&gt;</span><br><span class="line">&lt;/div&gt;</span><br><span class="line">一路上有你</span><br><span class="line">沧海一声笑</span><br><span class="line">往事随风</span><br><span class="line">光辉岁月</span><br><span class="line">记事本</span><br><span class="line">但愿人长久</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，<code>a</code>节点经过<code>sub()</code>方法处理后就没有了，然后再通过<code>findall()</code>方法直接提取即可。可以看到，在适当的时候，借助<code>sub()</code>方法可以起到事半功倍的效果。</p>
                  <h2 id="6-compile"><a href="#6-compile" class="headerlink" title="6. compile()"></a>6. <code>compile()</code></h2>
                  <p>前面所讲的方法都是用来处理字符串的方法，最后再介绍一下<code>compile()</code>方法，这个方法可以将正则字符串编译成正则表达式对象，以便在后面的匹配中复用。示例代码如下：</p>
                  <figure class="highlight lua">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import re</span><br><span class="line"></span><br><span class="line">content1 = <span class="string">'2016-12-15 12:00'</span></span><br><span class="line">content2 = <span class="string">'2016-12-17 12:55'</span></span><br><span class="line">content3 = <span class="string">'2016-12-22 13:21'</span></span><br><span class="line">pattern = re.compile(<span class="string">'\d&#123;2&#125;:\d&#123;2&#125;'</span>)</span><br><span class="line">result1 = re.<span class="built_in">sub</span>(pattern, <span class="string">''</span>, content1)</span><br><span class="line">result2 = re.<span class="built_in">sub</span>(pattern, <span class="string">''</span>, content2)</span><br><span class="line">result3 = re.<span class="built_in">sub</span>(pattern, <span class="string">''</span>, content3)</span><br><span class="line"><span class="built_in">print</span>(result1, result2, result3)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>例如，这里有3个日期，我们想分别将3个日期中的时间去掉，这时可以借助<code>sub()</code>方法。该方法的第一个参数是正则表达式，但是这里没有必要重复写3个同样的正则表达式，此时可以借助<code>compile()</code>方法将正则表达式编译成一个正则表达式对象，以便复用。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="number">2016</span><span class="number">-12</span><span class="number">-15</span>  <span class="number">2016</span><span class="number">-12</span><span class="number">-17</span>  <span class="number">2016</span><span class="number">-12</span><span class="number">-22</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，<code>compile()</code>还可以传入修饰符，例如<code>re.S</code>等修饰符，这样在<code>search()</code>、<code>findall()</code>等方法中就不需要额外传了。所以，<code>compile()</code>方法可以说是给正则表达式做了一层封装，以便我们更好地复用。</p>
                  <p>到此为止，正则表达式的基本用法就介绍完了，后面会通过具体的实例来讲解正则表达式的用法。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-27 14:32:40" itemprop="dateCreated datePublished" datetime="2018-01-27T14:32:40+08:00">2018-01-27</time>
                </span>
                <span id="/5530.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 3.3-正则表达式" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>12k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>11 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5523.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5523.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 3.2.2-高级用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>在前一节中，我们了解了requests的基本用法，如基本的GET、POST请求以及<code>Response</code>对象。本节中，我们再来了解下requests的一些高级用法，如文件上传、cookie设置、代理设置等。</p>
                  <h2 id="1-文件上传"><a href="#1-文件上传" class="headerlink" title="1. 文件上传"></a>1. 文件上传</h2>
                  <p>我们知道requests可以模拟提交一些数据。假如有的网站需要上传文件，我们也可以用它来实现，这非常简单，示例如下：</p>
                  <figure class="highlight vim">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line"><span class="keyword">files</span> = &#123;<span class="string">'file'</span>: <span class="keyword">open</span>(<span class="string">'favicon.ico'</span>, <span class="string">'rb'</span>)&#125;</span><br><span class="line">r = requests.post(<span class="string">"http://httpbin.org/post"</span>, <span class="keyword">files</span>=<span class="keyword">files</span>)</span><br><span class="line"><span class="keyword">print</span>(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>在前一节中我们保存了一个文件favicon.ico，这次用它来模拟文件上传的过程。需要注意的是，favicon.ico需要和当前脚本在同一目录下。如果有其他文件，当然也可以使用其他文件来上传，更改下代码即可。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"data"</span>: <span class="string">""</span>, </span><br><span class="line">  <span class="attr">"files"</span>: &#123;</span><br><span class="line">    <span class="attr">"file"</span>: <span class="string">"data:application/octet-stream;base64,AAAAAA...="</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"form"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"*/*"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="attr">"Content-Length"</span>: <span class="string">"6665"</span>, </span><br><span class="line">    <span class="attr">"Content-Type"</span>: <span class="string">"multipart/form-data; boundary=809f80b1a2974132b133ade1a8e8e058"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"python-requests/2.10.0"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"json"</span>: <span class="literal">null</span>, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"60.207.237.16"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/post"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>以上省略部分内容，这个网站会返回响应，里面包含<code>files</code>这个字段，而<code>form</code>字段是空的，这证明文件上传部分会单独有一个<code>files</code>字段来标识。</p>
                  <h2 id="2-Cookies"><a href="#2-Cookies" class="headerlink" title="2. Cookies"></a>2. Cookies</h2>
                  <p>前面我们使用urllib处理过Cookies，写法比较复杂，而有了requests，获取和设置Cookies只需一步即可完成。</p>
                  <p>我们先用一个实例看一下获取Cookies的过程：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="built_in">get</span>(<span class="string">"https://www.baidu.com"</span>)</span><br><span class="line"><span class="built_in">print</span>(r.cookies)</span><br><span class="line"><span class="keyword">for</span> <span class="built_in">key</span>, value in r.cookies.items():</span><br><span class="line">    <span class="built_in">print</span>(<span class="built_in">key</span> + <span class="string">'='</span> + value)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight fsharp">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;RequestsCookieJar<span class="meta">[&lt;Cookie BDORZ=27315 for .baidu.com/&gt;, &lt;Cookie __bsi=13533594356813414194_00_14_N_N_2_0303_C02F_N_N_N_0 for .www.baidu.com/&gt;]</span>&gt;</span><br><span class="line">BDORZ=<span class="number">27315</span></span><br><span class="line">__bsi=<span class="number">13533594356813414194</span>_00_14_N_N_2_0303_C02F_N_N_N_0</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先调用<code>cookies</code>属性即可成功得到Cookies，可以发现它是<code>RequestCookieJar</code>类型。然后用<code>items()</code>方法将其转化为元组组成的列表，遍历输出每一个Cookie的名称和值，实现Cookie的遍历解析。</p>
                  <p>当然，我们也可以直接用Cookie来维持登录状态，下面以知乎为例来说明。首先登录知乎，将<code>Headers</code>中的<code>Cookie</code>内容复制下来，如图3-6所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-6.png" alt="">图3-6 <code>Cookie</code></p>
                  <p>这里可以替换成你自己的<code>Cookie</code>，将其设置到<code>Headers</code>里面，然后发送请求，示例如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    'Cookie': 'q_c1=<span class="number">3165</span>3b264a074fc9a<span class="number">5781</span>6d1ea93ed8b|<span class="number">147427393800</span>0|<span class="number">147427393800</span>0; d_c0="AGDAs254kAqPTr6NW1U3XTLFzKhMPQ6H_nc=|<span class="number">1474273938</span>"; __utmv=<span class="number">51854390.10</span>0-1|2=registration_date=<span class="number">20130902</span>=1^3=entry_date=<span class="number">20130902</span>=1;a_t="2.0AACAfbwdAAAXAAAAso0QWAAAgH28HQAAAGDAs254kAoXAAAAYQJVTQ4FCVgA360us8BAklzLYNEHUd6kmHtRQX5a6hiZxKCynnycerLQ3gIkoJLOCQ==";z_c0=Mi4wQUFDQWZid2RBQUFBWU1DemJuaVFDaGNBQUFCaEFsVk5EZ1VKV0FEZnJTNnp3RUNTWE10ZzBRZFIzcVNZZTFGQmZn|<span class="number">1474887858</span>|64b4d<span class="number">4234</span>a21de774c42c837fe0b672fdb<span class="number">5763</span>b0',</span><br><span class="line">    'Host': 'www.zhihu.com',</span><br><span class="line">    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.<span class="number">2785.11</span>6 Safari/537.36',</span><br><span class="line">&#125;</span><br><span class="line">r = requests.get('https://www.zhihu.com', headers=headers)</span><br><span class="line">print(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们发现，结果中包含了登录后的结果，如图3-7所示，这证明登录成功。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-7.jpg" alt="">图3-7 运行结果</p>
                  <p>当然，你也可以通过<code>cookies</code>参数来设置，不过这样就需要构造<code>RequestsCookieJar</code>对象，而且需要分割一下<code>cookies</code>。这相对烦琐，不过效果是相同的，示例如下：</p>
                  <figure class="highlight 1c">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">cookies = 'q_c1=<span class="number">3165</span>3b264a074fc9a<span class="number">5781</span>6d1ea93ed8b|<span class="number">147427393800</span>0|<span class="number">147427393800</span>0; d_c0="AGDAs254kAqPTr6NW1U3XTLFzKhMPQ6H_nc=|<span class="number">1474273938</span>"; __utmv=<span class="number">51854390.10</span>0-1|2=registration_date=<span class="number">20130902</span>=1^3=entry_date=<span class="number">20130902</span>=1;a_t="2.0AACAfbwdAAAXAAAAso0QWAAAgH28HQAAAGDAs254kAoXAAAAYQJVTQ4FCVgA360us8BAklzLYNEHUd6kmHtRQX5a6hiZxKCynnycerLQ3gIkoJLOCQ==";z_c0=Mi4wQUFDQWZid2RBQUFBWU1DemJuaVFDaGNBQUFCaEFsVk5EZ1VKV0FEZnJTNnp3RUNTWE10ZzBRZFIzcVNZZTFGQmZn|<span class="number">1474887858</span>|64b4d<span class="number">4234</span>a21de774c42c837fe0b672fdb<span class="number">5763</span>b0'</span><br><span class="line">jar = requests.cookies.RequestsCookieJar()</span><br><span class="line">headers = &#123;</span><br><span class="line">    'Host': 'www.zhihu.com',</span><br><span class="line">    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.<span class="number">2785.11</span>6 Safari/537.36'</span><br><span class="line">&#125;</span><br><span class="line">for cookie in cookies.split(';'):</span><br><span class="line">    key, value = cookie.split('=', <span class="number">1</span>)</span><br><span class="line">    jar.set(key, value)</span><br><span class="line">r = requests.get(<span class="string">"http://www.zhihu.com"</span>, cookies=jar, headers=headers)</span><br><span class="line">print(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们首先新建了一个<code>RequestCookieJar</code>对象，然后将复制下来的<code>cookies</code>利用<code>split()</code>方法分割，接着利用<code>set()</code>方法设置好每个Cookie的<code>key</code>和<code>value</code>，然后通过调用requests的<code>get()</code>方法并传递给<code>cookies</code>参数即可。当然，由于知乎本身的限制，<code>headers</code>参数也不能少，只不过不需要在原来的<code>headers</code>参数里面设置<code>cookie</code>字段了。</p>
                  <p>测试后，发现同样可以正常登录知乎。</p>
                  <h2 id="3-会话维持"><a href="#3-会话维持" class="headerlink" title="3. 会话维持"></a>3. 会话维持</h2>
                  <p>在requests中，如果直接利用<code>get()</code>或<code>post()</code>等方法的确可以做到模拟网页的请求，但是这实际上是相当于不同的会话，也就是说相当于你用了两个浏览器打开了不同的页面。</p>
                  <p>设想这样一个场景，第一个请求利用<code>post()</code>方法登录了某个网站，第二次想获取成功登录后的自己的个人信息，你又用了一次<code>get()</code>方法去请求个人信息页面。实际上，这相当于打开了两个浏览器，是两个完全不相关的会话，能成功获取个人信息吗？那当然不能。</p>
                  <p>有小伙伴可能说了，我在两次请求时设置一样的<code>cookies</code>不就行了？可以，但这样做起来显得很烦琐，我们有更简单的解决方法。</p>
                  <p>其实解决这个问题的主要方法就是维持同一个会话，也就是相当于打开一个新的浏览器选项卡而不是新开一个浏览器。但是我又不想每次设置<code>cookies</code>，那该怎么办呢？这时候就有了新的利器——<code>Session</code>对象。</p>
                  <p>利用它，我们可以方便地维护一个会话，而且不用担心<code>cookies</code>的问题，它会帮我们自动处理好。示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">requests.<span class="builtin-name">get</span>(<span class="string">'http://httpbin.org/cookies/set/number/123456789'</span>)</span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">'http://httpbin.org/cookies'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们请求了一个测试网址<a href="http://httpbin.org/cookies/set/number/123456789" target="_blank" rel="noopener">http://httpbin.org/cookies/set/number/123456789</a>。请求这个网址时，可以设置一个cookie，名称叫作number，内容是123456789，随后又请求了<a href="http://httpbin.org/cookies" target="_blank" rel="noopener">http://httpbin.org/cookies</a>，此网址可以获取当前的Cookies。</p>
                  <p>这样能成功获取到设置的Cookies吗？试试看。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"cookies"</span>: &#123;&#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这并不行。我们再用<code>Session</code>试试看：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">s = requests.Session()</span><br><span class="line">s.<span class="builtin-name">get</span>(<span class="string">'http://httpbin.org/cookies/set/number/123456789'</span>)</span><br><span class="line">r = s.<span class="builtin-name">get</span>(<span class="string">'http://httpbin.org/cookies'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>再看下运行结果：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"cookies"</span>: &#123;</span><br><span class="line">    <span class="attr">"number"</span>: <span class="string">"123456789"</span></span><br><span class="line">  &#125;</span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>成功获取！这下能体会到同一个会话和不同会话的区别了吧！</p>
                  <p>所以，利用<code>Session</code>，可以做到模拟同一个会话而不用担心Cookies的问题。它通常用于模拟登录成功之后再进行下一步的操作。</p>
                  <p><code>Session</code>在平常用得非常广泛，可以用于模拟在一个浏览器中打开同一站点的不同页面，后面会有专门的章节来讲解这部分内容。</p>
                  <h2 id="4-SSL证书验证"><a href="#4-SSL证书验证" class="headerlink" title="4. SSL证书验证"></a>4. SSL证书验证</h2>
                  <p>此外，requests还提供了证书验证的功能。当发送HTTP请求的时候，它会检查SSL证书，我们可以使用<code>verify</code>参数控制是否检查此证书。其实如果不加<code>verify</code>参数的话，默认是<code>True</code>，会自动验证。</p>
                  <p>前面我们提到过，12306的证书没有被官方CA机构信任，会出现证书验证错误的结果。我们现在访问它，都可以看到一个证书问题的页面，如图3-8所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-8.png" alt="">图3-8 错误页面</p>
                  <p>现在我们用requests来测试一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">response = requests.<span class="builtin-name">get</span>(<span class="string">'https://www.12306.cn'</span>)</span><br><span class="line"><span class="builtin-name">print</span>(response.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight css">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="selector-tag">requests</span><span class="selector-class">.exceptions</span><span class="selector-class">.SSLError</span>: ("<span class="selector-tag">bad</span> <span class="selector-tag">handshake</span>: <span class="selector-tag">Error</span>(<span class="selector-attr">[(<span class="string">'SSL routines'</span>, <span class="string">'tls_process_server_certificate'</span>, <span class="string">'certificate verify failed'</span>)]</span>,)",)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里提示一个错误<code>SSLError</code>，表示证书验证错误。所以，如果请求一个HTTPS站点，但是证书验证错误的页面时，就会报这样的错误，那么如何避免这个错误呢？很简单，把<code>verify</code>参数设置为<code>False</code>即可。相关代码如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">response = requests.<span class="builtin-name">get</span>(<span class="string">'https://www.12306.cn'</span>, <span class="attribute">verify</span>=<span class="literal">False</span>)</span><br><span class="line"><span class="builtin-name">print</span>(response.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样就会打印出请求成功的状态码：</p>
                  <figure class="highlight crystal">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">/usr/local/<span class="class"><span class="keyword">lib</span>/<span class="title">python3</span>.6/<span class="title">site</span>-<span class="title">packages</span>/<span class="title">urllib3</span>/<span class="title">connectionpool</span>.<span class="title">py</span>:852: <span class="title">InsecureRequestWarning</span>: <span class="title">Unverified</span> <span class="title">HTTPS</span> <span class="title">request</span> <span class="title">is</span> <span class="title">being</span> <span class="title">made</span>. <span class="title">Adding</span> <span class="title">certificate</span> <span class="title">verification</span> <span class="title">is</span> <span class="title">strongly</span> <span class="title">advised</span>. <span class="title">See</span>: <span class="title">https</span>://<span class="title">urllib3</span>.<span class="title">readthedocs</span>.<span class="title">io</span>/<span class="title">en</span>/<span class="title">latest</span>/<span class="title">advanced</span>-<span class="title">usage</span>.<span class="title">html</span><span class="comment">#ssl-warnings</span></span></span><br><span class="line">  InsecureRequestWarning)</span><br><span class="line"><span class="number">200</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>不过我们发现报了一个警告，它建议我们给它指定证书。我们可以通过设置忽略警告的方式来屏蔽这个警告：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"><span class="keyword">from</span> requests.packages import urllib3</span><br><span class="line"></span><br><span class="line">urllib3.disable_warnings()</span><br><span class="line">response = requests.<span class="builtin-name">get</span>(<span class="string">'https://www.12306.cn'</span>, <span class="attribute">verify</span>=<span class="literal">False</span>)</span><br><span class="line"><span class="builtin-name">print</span>(response.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或者通过捕获警告到日志的方式忽略警告：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import logging</span><br><span class="line">import requests</span><br><span class="line">logging.captureWarnings(<span class="literal">True</span>)</span><br><span class="line">response = requests.<span class="builtin-name">get</span>(<span class="string">'https://www.12306.cn'</span>, <span class="attribute">verify</span>=<span class="literal">False</span>)</span><br><span class="line"><span class="builtin-name">print</span>(response.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，我们也可以指定一个本地证书用作客户端证书，这可以是单个文件（包含密钥和证书）或一个包含两个文件路径的元组：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">response = requests.<span class="builtin-name">get</span>(<span class="string">'https://www.12306.cn'</span>, cert=(<span class="string">'/path/server.crt'</span>, <span class="string">'/path/key'</span>))</span><br><span class="line"><span class="builtin-name">print</span>(response.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，上面的代码是演示实例，我们需要有crt和key文件，并且指定它们的路径。注意，本地私有证书的<code>key</code>必须是解密状态，加密状态的<code>key</code>是不支持的。</p>
                  <h2 id="5-代理设置"><a href="#5-代理设置" class="headerlink" title="5. 代理设置"></a>5. 代理设置</h2>
                  <p>对于某些网站，在测试的时候请求几次，能正常获取内容。但是一旦开始大规模爬取，对于大规模且频繁的请求，网站可能会弹出验证码，或者跳转到登录认证页面，更甚者可能会直接封禁客户端的IP，导致一定时间段内无法访问。</p>
                  <p>那么，为了防止这种情况发生，我们需要设置代理来解决这个问题，这就需要用到<code>proxies</code>参数。可以用这样的方式设置：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">proxies = &#123;</span><br><span class="line">  <span class="string">"http"</span>: <span class="string">"http://10.10.1.10:3128"</span>,</span><br><span class="line">  <span class="string">"https"</span>: <span class="string">"http://10.10.1.10:1080"</span>,</span><br><span class="line">&#125;</span><br><span class="line"></span><br><span class="line">requests.<span class="builtin-name">get</span>(<span class="string">"https://www.taobao.com"</span>, <span class="attribute">proxies</span>=proxies)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，直接运行这个实例可能不行，因为这个代理可能是无效的，请换成自己的有效代理试验一下。</p>
                  <p>若代理需要使用HTTP Basic Auth，可以使用类似<a href="http://user:password@host:port这样的语法来设置代理，示例如下：">http://user:password@host:port这样的语法来设置代理，示例如下：</a></p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">proxies = &#123;</span><br><span class="line">    <span class="string">"http"</span>: <span class="string">"http://user:password@10.10.1.10:3128/"</span>,</span><br><span class="line">&#125;</span><br><span class="line">requests.<span class="builtin-name">get</span>(<span class="string">"https://www.taobao.com"</span>, <span class="attribute">proxies</span>=proxies)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>除了基本的HTTP代理外，requests还支持SOCKS协议的代理。</p>
                  <p>首先，需要安装socks这个库：</p>
                  <figure class="highlight nginx">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attribute">pip3</span> install <span class="string">'requests[socks]'</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>然后就可以使用SOCKS协议代理了，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">proxies = &#123;</span><br><span class="line">    <span class="string">'http'</span>: <span class="string">'socks5://user:password@host:port'</span>,</span><br><span class="line">    <span class="string">'https'</span>: <span class="string">'socks5://user:password@host:port'</span></span><br><span class="line">&#125;</span><br><span class="line">requests.<span class="builtin-name">get</span>(<span class="string">"https://www.taobao.com"</span>, <span class="attribute">proxies</span>=proxies)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="6-超时设置"><a href="#6-超时设置" class="headerlink" title="6. 超时设置"></a>6. 超时设置</h2>
                  <p>在本机网络状况不好或者服务器网络响应太慢甚至无响应时，我们可能会等待特别久的时间才可能收到响应，甚至到最后收不到响应而报错。为了防止服务器不能及时响应，应该设置一个超时时间，即超过了这个时间还没有得到响应，那就报错。这需要用到<code>timeout</code>参数。这个时间的计算是发出请求到服务器返回响应的时间。示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">"https://www.taobao.com"</span>, timeout = 1)</span><br><span class="line"><span class="builtin-name">print</span>(r.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过这样的方式，我们可以将超时时间设置为1秒，如果1秒内没有响应，那就抛出异常。</p>
                  <p>实际上，请求分为两个阶段，即连接（connect）和读取（read）。</p>
                  <p>上面设置的<code>timeout</code>将用作连接和读取这二者的<code>timeout</code>总和。</p>
                  <p>如果要分别指定，就可以传入一个元组：</p>
                  <figure class="highlight angelscript">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">r = requests.<span class="keyword">get</span>(<span class="string">'https://www.taobao.com'</span>, timeout=(<span class="number">5</span>,<span class="number">11</span>, <span class="number">30</span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果想永久等待，可以直接将<code>timeout</code>设置为<code>None</code>，或者不设置直接留空，因为默认是<code>None</code>。这样的话，如果服务器还在运行，但是响应特别慢，那就慢慢等吧，它永远不会返回超时错误的。其用法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">'https://www.taobao.com'</span>, <span class="attribute">timeout</span>=None)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>或直接不加参数：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">r</span> = requests.get(<span class="string">'https://www.taobao.com'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <h2 id="7-身份认证"><a href="#7-身份认证" class="headerlink" title="7. 身份认证"></a>7. 身份认证</h2>
                  <p>在访问网站时，我们可能会遇到这样的认证页面，如图3-9所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-9.jpg" alt="">图3-9 认证页面</p>
                  <p>此时可以使用requests自带的身份认证功能，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"><span class="keyword">from</span> requests.auth import HTTPBasicAuth</span><br><span class="line"></span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">'http://localhost:5000'</span>, <span class="attribute">auth</span>=HTTPBasicAuth('username', <span class="string">'password'</span>))</span><br><span class="line"><span class="builtin-name">print</span>(r.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>如果用户名和密码正确的话，请求时就会自动认证成功，会返回200状态码，如果认证失败，则返回401状态码。</p>
                  <p>当然，如果参数都传一个<code>HTTPBasicAuth</code>类，就显得有点烦琐了，所以requests提供了一个更简单的写法，可以直接传一个元组，它会默认使用<code>HTTPBasicAuth</code>这个类来认证。</p>
                  <p>所以上面的代码可以直接简写如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">'http://localhost:5000'</span>, auth=(<span class="string">'username'</span>, <span class="string">'password'</span>))</span><br><span class="line"><span class="builtin-name">print</span>(r.status_code)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>此外，requests还提供了其他认证方式，如OAuth认证，不过此时需要安装oauth包，安装命令如下：</p>
                  <figure class="highlight cmake">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">pip3 <span class="keyword">install</span> requests_oauthlib</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>使用OAuth1认证的方法如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"><span class="keyword">from</span> requests_oauthlib import OAuth1</span><br><span class="line"></span><br><span class="line">url = <span class="string">'https://api.twitter.com/1.1/account/verify_credentials.json'</span></span><br><span class="line">auth = OAuth1(<span class="string">'YOUR_APP_KEY'</span>, <span class="string">'YOUR_APP_SECRET'</span>,</span><br><span class="line">              <span class="string">'USER_OAUTH_TOKEN'</span>, <span class="string">'USER_OAUTH_TOKEN_SECRET'</span>)</span><br><span class="line">requests.<span class="builtin-name">get</span>(url, <span class="attribute">auth</span>=auth)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>更多详细的功能可以参考requests_oauthlib的官方文档<a href="https://requests-oauthlib.readthedocs.org/" target="_blank" rel="noopener">https://requests-oauthlib.readthedocs.org/</a>，在此不再赘述了。</p>
                  <h2 id="8-Prepared-Request"><a href="#8-Prepared-Request" class="headerlink" title="8. Prepared Request"></a>8. Prepared Request</h2>
                  <p>前面介绍urllib时，我们可以将请求表示为数据结构，其中各个参数都可以通过一个<code>Request</code>对象来表示。这在requests里同样可以做到，这个数据结构就叫Prepared Request。我们用实例看一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">from</span> requests import Request, Session</span><br><span class="line"></span><br><span class="line">url = <span class="string">'http://httpbin.org/post'</span></span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'germey'</span></span><br><span class="line">&#125;</span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36'</span></span><br><span class="line">&#125;</span><br><span class="line">s = Session()</span><br><span class="line">req = Request(<span class="string">'POST'</span>, url, <span class="attribute">data</span>=data, <span class="attribute">headers</span>=headers)</span><br><span class="line">prepped = s.prepare_request(req)</span><br><span class="line">r = s.send(prepped)</span><br><span class="line"><span class="builtin-name">print</span>(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们引入了<code>Request</code>，然后用<code>url</code>、<code>data</code>和<code>headers</code>参数构造了一个<code>Request</code>对象，这时需要再调用<code>Session</code>的<code>prepare_request()</code>方法将其转换为一个Prepared Request对象，然后调用<code>send()</code>方法发送即可，运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"data"</span>: <span class="string">""</span>, </span><br><span class="line">  <span class="attr">"files"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"form"</span>: &#123;</span><br><span class="line">    <span class="attr">"name"</span>: <span class="string">"germey"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"*/*"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="attr">"Connection"</span>: <span class="string">"close"</span>, </span><br><span class="line">    <span class="attr">"Content-Length"</span>: <span class="string">"11"</span>, </span><br><span class="line">    <span class="attr">"Content-Type"</span>: <span class="string">"application/x-www-form-urlencoded"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.116 Safari/537.36"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"json"</span>: <span class="literal">null</span>, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"182.32.203.166"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/post"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以看到，我们达到了同样的POST请求效果。</p>
                  <p>有了<code>Request</code>这个对象，就可以将请求当作独立的对象来看待，这样在进行队列调度时会非常方便。后面我们会用它来构造一个<code>Request</code>队列。</p>
                  <p>本节讲解了requests的一些高级用法，这些用法在后面实战部分会经常用到，需要熟练掌握。更多的用法可以参考requests的官方文档：<a href="http://docs.python-requests.org/" target="_blank" rel="noopener">http://docs.python-requests.org/</a>。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-27 14:26:20" itemprop="dateCreated datePublished" datetime="2018-01-27T14:26:20+08:00">2018-01-27</time>
                </span>
                <span id="/5523.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 3.2.2-高级用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>10k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>9 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5517.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5517.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 3.2.1-基本用法</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <h2 id="1-准备工作"><a href="#1-准备工作" class="headerlink" title="1. 准备工作"></a>1. 准备工作</h2>
                  <p>在开始之前，请确保已经正确安装好了requests库。如果没有安装，可以参考1.2.1节安装。</p>
                  <h2 id="2-实例引入"><a href="#2-实例引入" class="headerlink" title="2. 实例引入"></a>2. 实例引入</h2>
                  <p>urllib库中的<code>urlopen()</code>方法实际上是以GET方式请求网页，而requests中相应的方法就是<code>get()</code>方法，是不是感觉表达更明确一些？下面通过实例来看一下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.get(<span class="string">'https://www.baidu.com/'</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(type(r)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(r.status_code)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(type(r.text)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(r.text)</span></span></span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(r.cookies)</span></span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight django">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">requests.models.Response</span>'&gt;</span></span></span><br><span class="line"><span class="xml">200</span></span><br><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">class</span> '<span class="attr">str</span>'&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">html</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">head</span>&gt;</span></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">script</span>&gt;</span></span></span><br><span class="line"><span class="xml">        location.replace(location.href.replace("https://","http://"));</span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;/<span class="name">script</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;/<span class="name">head</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">body</span>&gt;</span></span></span><br><span class="line"><span class="xml">    <span class="tag">&lt;<span class="name">noscript</span>&gt;</span><span class="tag">&lt;<span class="name">meta</span> <span class="attr">http-equiv</span>=<span class="string">"refresh"</span> <span class="attr">content</span>=<span class="string">"0;url=http://www.baidu.com/"</span>&gt;</span><span class="tag">&lt;/<span class="name">noscript</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;/<span class="name">body</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span></span><br><span class="line"><span class="xml"><span class="tag">&lt;<span class="name">RequestsCookieJar[</span>&lt;<span class="attr">Cookie</span> <span class="attr">BIDUPSID</span>=<span class="string">992C3B26F4C4D09505C5E959D5FBC005</span> <span class="attr">for</span> <span class="attr">.baidu.com</span>/&gt;</span>, <span class="tag">&lt;<span class="name">Cookie</span> <span class="attr">PSTM</span>=<span class="string">1472227535</span> <span class="attr">for</span> <span class="attr">.baidu.com</span>/&gt;</span>, <span class="tag">&lt;<span class="name">Cookie</span> <span class="attr">__bsi</span>=<span class="string">15304754498609545148_00_40_N_N_2_0303_C02F_N_N_N_0</span> <span class="attr">for</span> <span class="attr">.www.baidu.com</span>/&gt;</span>, <span class="tag">&lt;<span class="name">Cookie</span> <span class="attr">BD_NOT_HTTPS</span>=<span class="string">1</span> <span class="attr">for</span> <span class="attr">www.baidu.com</span>/&gt;</span>]&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们调用<code>get()</code>方法实现与<code>urlopen()</code>相同的操作，得到一个<code>Response</code>对象，然后分别输出了<code>Response</code>的类型、状态码、响应体的类型、内容以及Cookies。</p>
                  <p>通过运行结果可以发现，它的返回类型是<code>requests.models.Response</code>，响应体的类型是字符串<code>str</code>，Cookies的类型是<code>RequestsCookieJar</code>。</p>
                  <p>使用<code>get()</code>方法成功实现一个GET请求，这倒不算什么，更方便之处在于其他的请求类型依然可以用一句话来完成，示例如下：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">r</span> = requests.post(<span class="string">'http://httpbin.org/post'</span>)</span><br><span class="line"><span class="attr">r</span> = requests.put(<span class="string">'http://httpbin.org/put'</span>)</span><br><span class="line"><span class="attr">r</span> = requests.delete(<span class="string">'http://httpbin.org/delete'</span>)</span><br><span class="line"><span class="attr">r</span> = requests.head(<span class="string">'http://httpbin.org/get'</span>)</span><br><span class="line"><span class="attr">r</span> = requests.options(<span class="string">'http://httpbin.org/get'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里分别用<code>post()</code>、<code>put()</code>、<code>delete()</code>等方法实现了POST、PUT、DELETE等请求。是不是比urllib简单太多了？</p>
                  <p>其实这只是冰山一角，更多的还在后面。</p>
                  <h2 id="3-GET请求"><a href="#3-GET请求" class="headerlink" title="3. GET请求"></a>3. GET请求</h2>
                  <p>HTTP中最常见的请求之一就是GET请求，下面首先来详细了解一下利用requests构建GET请求的方法。</p>
                  <h3 id="基本实例"><a href="#基本实例" class="headerlink" title="基本实例"></a>基本实例</h3>
                  <p>首先，构建一个最简单的GET请求，请求的链接为<a href="http://epub.ituring.com.cn/article/edit/[http://httpbin.org/get" target="_blank" rel="noopener">http://httpbin.org/get</a>，该网站会判断如果客户端发起的是GET请求的话，它返回相应的请求信息：</p>
                  <figure class="highlight processing">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="built_in">get</span>(<span class="string">'http://httpbin.org/get'</span>)</span><br><span class="line"><span class="built_in">print</span>(r.<span class="built_in">text</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"*/*"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"python-requests/2.10.0"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"122.4.215.33"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/get"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，我们成功发起了GET请求，返回结果中包含请求头、URL、IP等信息。</p>
                  <p>那么，对于GET请求，如果要附加额外的信息，一般怎样添加呢？比如现在想添加两个参数，其中<code>name</code>是<code>germey</code>，<code>age</code>是22。要构造这个请求链接，是不是要直接写成：</p>
                  <figure class="highlight ini">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="attr">r</span> = requests.get(<span class="string">'http://httpbin.org/get?name=germey&amp;age=22'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这样也可以，但是是不是有点不人性化呢？一般情况下，这种信息数据会用字典来存储。那么，怎样来构造这个链接呢？</p>
                  <p>这同样很简单，利用<code>params</code>这个参数就好了，示例如下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">data = &#123;</span><br><span class="line">    <span class="string">'name'</span>: <span class="string">'germey'</span>,</span><br><span class="line">    <span class="string">'age'</span>: 22</span><br><span class="line">&#125;</span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">"http://httpbin.org/get"</span>, <span class="attribute">params</span>=data)</span><br><span class="line"><span class="builtin-name">print</span>(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"args"</span>: &#123;</span><br><span class="line">    <span class="attr">"age"</span>: <span class="string">"22"</span>, </span><br><span class="line">    <span class="attr">"name"</span>: <span class="string">"germey"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"*/*"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"python-requests/2.10.0"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"122.4.215.33"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/get?age=22&amp;name=germey"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>通过运行结果可以判断，请求的链接自动被构造成了：<a href="http://httpbin.org/get?age=22&amp;name=germey" target="_blank" rel="noopener">http://httpbin.org/get?age=22&amp;name=germey</a>。</p>
                  <p>另外，网页的返回类型实际上是<code>str</code>类型，但是它很特殊，是JSON格式的。所以，如果想直接解析返回结果，得到一个字典格式的话，可以直接调用<code>json()</code>方法。示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.get(<span class="string">"http://httpbin.org/get"</span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(type(r.text)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(r.json()</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(type(r.json()</span></span>))</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight ceylon">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="keyword">class</span> <span class="string">'str'</span>&gt;</span><br><span class="line">&#123;<span class="string">'headers'</span>: &#123;<span class="string">'Accept-Encoding'</span>: <span class="string">'gzip, deflate'</span>, <span class="string">'Accept'</span>: <span class="string">'*/*'</span>, <span class="string">'Host'</span>: <span class="string">'httpbin.org'</span>, <span class="string">'User-Agent'</span>: <span class="string">'python-requests/2.10.0'</span>&#125;, <span class="string">'url'</span>: <span class="string">'http://httpbin.org/get'</span>, <span class="string">'args'</span>: &#123;&#125;, <span class="string">'origin'</span>: <span class="string">'182.33.248.131'</span>&#125;</span><br><span class="line">&lt;<span class="keyword">class</span> <span class="string">'dict'</span>&gt;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，调用<code>json()</code>方法，就可以将返回结果是JSON格式的字符串转化为字典。</p>
                  <p>但需要注意的书，如果返回结果不是JSON格式，便会出现解析错误，抛出<code>json.decoder.JSONDecodeError</code>异常。</p>
                  <h3 id="抓取网页"><a href="#抓取网页" class="headerlink" title="抓取网页"></a>抓取网页</h3>
                  <p>上面的请求链接返回的是JSON形式的字符串，那么如果请求普通的网页，则肯定能获得相应的内容了。下面以“知乎”→“发现”页面为例来看一下：</p>
                  <figure class="highlight roboconf">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">import</span> re</span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    '<span class="attribute">User-Agent'</span>: 'Mozilla/5.0 (Macintosh; <span class="attribute">Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'</span></span><br><span class="line"><span class="attribute">&#125;</span></span><br><span class="line"><span class="attribute">r = requests.get("https</span>://www<span class="variable">.zhihu</span><span class="variable">.com</span>/explore", headers=headers)</span><br><span class="line">pattern = re<span class="variable">.compile</span>('explore-feed.*?question_link.*?&gt;(.*?)&lt;/a&gt;', re<span class="variable">.S</span>)</span><br><span class="line">titles = re<span class="variable">.findall</span>(pattern, r<span class="variable">.text</span>)</span><br><span class="line">print(titles)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里我们加入了<code>headers</code>信息，其中包含了<code>User-Agent</code>字段信息，也就是浏览器标识信息。如果不加这个，知乎会禁止抓取。</p>
                  <p>接下来我们用到了最基础的正则表达式来匹配出所有的问题内容。关于正则表达式的相关内容，我们会在3.3节中详细介绍，这里作为实例来配合讲解。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight scheme">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">['\n为什么很多人喜欢提及「拉丁语系」这个词？\n', '\n在没有水的情况下水系宝可梦如何战斗？\n', '\n有哪些经验可以送给 Kindle 新人？\n', '\n谷歌的广告业务是如何赚钱的？\n', '\n程序员该学习什么，能在上学期间挣钱？\n', '\n有哪些原本只是一个小消息，但回看发现是个惊天大新闻的例子？\n', '\n如何评价今敏？\n', '\n源氏是怎么把那么长的刀从背后拔出来的？\n', '\n年轻时得了绝症或大病是怎样的感受？\n', '\n年轻时得了绝症或大病是怎样的感受？\n']</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>我们发现，这里成功提取出了所有的问题内容。</p>
                  <h3 id="抓取二进制数据"><a href="#抓取二进制数据" class="headerlink" title="抓取二进制数据"></a>抓取二进制数据</h3>
                  <p>在上面的例子中，我们抓取的是知乎的一个页面，实际上它返回的是一个HTML文档。如果想抓去图片、音频、视频等文件，应该怎么办呢？</p>
                  <p>图片、音频、视频这些文件本质上都是由二进制码组成的，由于有特定的保存格式和对应的解析方式，我们才可以看到这些形形色色的多媒体。所以，想要抓取它们，就要拿到它们的二进制码。</p>
                  <p>下面以GitHub的站点图标为例来看一下：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">"https://github.com/favicon.ico"</span>)</span><br><span class="line"><span class="builtin-name">print</span>(r.text)</span><br><span class="line"><span class="builtin-name">print</span>(r.content)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里抓取的内容是站点图标，也就是在浏览器每一个标签上显示的小图标，如图3-3所示。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-3.png" alt="">图3-3 站点图标</p>
                  <p>这里打印了<code>Response</code>对象的两个属性，一个是<code>text</code>，另一个是<code>content</code>。</p>
                  <p>运行结果如图3-4所示，其中前两行是<code>r.text</code>的结果，最后一行是<code>r.content</code>的结果。</p>
                  <p><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-4.png" alt="">图3-4 运行结果</p>
                  <p>可以注意到，前者出现了乱码，后者结果前带有一个<code>b</code>，这代表是<code>bytes</code>类型的数据。由于图片是二进制数据，所以前者在打印时转化为<code>str</code>类型，也就是图片直接转化为字符串，这理所当然会出现乱码。</p>
                  <p>接着，我们将刚才提取到的图片保存下来：</p>
                  <figure class="highlight livecodeserver">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="built_in">get</span>(<span class="string">"https://github.com/favicon.ico"</span>)</span><br><span class="line"><span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">'favicon.ico'</span>, <span class="string">'wb'</span>) <span class="keyword">as</span> f:</span><br><span class="line">    f.<span class="built_in">write</span>(r.content)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里用了<code>open()</code>方法，它的第一个参数是文件名称，第二个参数代表以二进制写的形式打开，可以向文件里写入二进制数据。</p>
                  <p>运行结束之后，可以发现在文件夹中出现了名为favicon.ico的图标，如图3-5所示。</p>
                  <p><a href="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-5.ico" target="_blank" rel="noopener"><img src="https://qiniu.cuiqingcai.com/wp-content/uploads/2018/02/3-5.ico" alt=""></a>图3-5 图标</p>
                  <p>同样地，音频和视频文件也可以用这种方法获取。</p>
                  <h3 id="添加headers"><a href="#添加headers" class="headerlink" title="添加headers"></a>添加<code>headers</code></h3>
                  <p>与<code>urllib.request</code>一样，我们也可以通过<code>headers</code>参数来传递头信息。</p>
                  <p>比如，在上面“知乎”的例子中，如果不传递<code>headers</code>，就不能正常请求：</p>
                  <figure class="highlight arduino">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line">r = requests.<span class="built_in">get</span>(<span class="string">"https://www.zhihu.com/explore"</span>)</span><br><span class="line"><span class="built_in">print</span>(r.<span class="built_in">text</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果如下：</p>
                  <figure class="highlight xml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="tag">&lt;<span class="name">html</span>&gt;</span><span class="tag">&lt;<span class="name">body</span>&gt;</span><span class="tag">&lt;<span class="name">h1</span>&gt;</span>500 Server Error<span class="tag">&lt;/<span class="name">h1</span>&gt;</span></span><br><span class="line">An internal server error occured.</span><br><span class="line"><span class="tag">&lt;/<span class="name">body</span>&gt;</span><span class="tag">&lt;/<span class="name">html</span>&gt;</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>但如果加上<code>headers</code>并加上<code>User-Agent</code>信息，那就没问题了：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">headers = &#123;</span><br><span class="line">    <span class="string">'User-Agent'</span>: <span class="string">'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/52.0.2743.116 Safari/537.36'</span></span><br><span class="line">&#125;</span><br><span class="line">r = requests.<span class="builtin-name">get</span>(<span class="string">"https://www.zhihu.com/explore"</span>, <span class="attribute">headers</span>=headers)</span><br><span class="line"><span class="builtin-name">print</span>(r.text)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，我们可以在<code>headers</code>这个参数中任意添加其他的字段信息。</p>
                  <h2 id="4-POST请求"><a href="#4-POST请求" class="headerlink" title="4. POST请求"></a>4. POST请求</h2>
                  <p>前面我们了解了最基本的GET请求，另外一种比较常见的请求方式是POST。使用<code>requests</code>实现POST请求同样非常简单，示例如下：</p>
                  <figure class="highlight xl">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="keyword">data</span> = &#123;<span class="string">'name'</span>: <span class="string">'germey'</span>, <span class="string">'age'</span>: <span class="string">'22'</span>&#125;</span><br><span class="line">r = requests.post(<span class="string">"http://httpbin.org/post"</span>, <span class="keyword">data</span>=<span class="keyword">data</span>)</span><br><span class="line">print(r.<span class="keyword">text</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里还是请求<a href="http://httpbin.org/post" target="_blank" rel="noopener">http://httpbin.org/post</a>，该网站可以判断如果请求是POST方式，就把相关请求信息返回。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight json">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&#123;</span><br><span class="line">  <span class="attr">"args"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"data"</span>: <span class="string">""</span>, </span><br><span class="line">  <span class="attr">"files"</span>: &#123;&#125;, </span><br><span class="line">  <span class="attr">"form"</span>: &#123;</span><br><span class="line">    <span class="attr">"age"</span>: <span class="string">"22"</span>, </span><br><span class="line">    <span class="attr">"name"</span>: <span class="string">"germey"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"headers"</span>: &#123;</span><br><span class="line">    <span class="attr">"Accept"</span>: <span class="string">"*/*"</span>, </span><br><span class="line">    <span class="attr">"Accept-Encoding"</span>: <span class="string">"gzip, deflate"</span>, </span><br><span class="line">    <span class="attr">"Content-Length"</span>: <span class="string">"18"</span>, </span><br><span class="line">    <span class="attr">"Content-Type"</span>: <span class="string">"application/x-www-form-urlencoded"</span>, </span><br><span class="line">    <span class="attr">"Host"</span>: <span class="string">"httpbin.org"</span>, </span><br><span class="line">    <span class="attr">"User-Agent"</span>: <span class="string">"python-requests/2.10.0"</span></span><br><span class="line">  &#125;, </span><br><span class="line">  <span class="attr">"json"</span>: <span class="literal">null</span>, </span><br><span class="line">  <span class="attr">"origin"</span>: <span class="string">"182.33.248.131"</span>, </span><br><span class="line">  <span class="attr">"url"</span>: <span class="string">"http://httpbin.org/post"</span></span><br><span class="line">&#125;</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>可以发现，我们成功获得了返回结果，其中<code>form</code>部分就是提交的数据，这就证明POST请求成功发送了。</p>
                  <h2 id="5-响应"><a href="#5-响应" class="headerlink" title="5. 响应"></a>5. 响应</h2>
                  <p>发送请求后，得到的自然就是响应。在上面的实例中，我们使用<code>text</code>和<code>content</code>获取了响应的内容。此外，还有很多属性和方法可以用来获取其他信息，比如状态码、响应头、Cookies等。示例如下：</p>
                  <figure class="highlight haskell">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="title">r</span> = requests.get('http://www.jianshu.com')</span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">r</span>.<span class="title">status_code</span>), r.status_code)</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">r</span>.<span class="title">headers</span>), r.headers)</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">r</span>.<span class="title">cookies</span>), r.cookies)</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">r</span>.<span class="title">url</span>), r.url)</span></span><br><span class="line"><span class="title">print</span>(<span class="class"><span class="keyword">type</span>(<span class="title">r</span>.<span class="title">history</span>), r.history)</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里分别打印输出<code>status_code</code>属性得到状态码，输出<code>headers</code>属性得到响应头，输出<code>cookies</code>属性得到Cookies，输出<code>url</code>属性得到URL，输出<code>history</code>属性得到请求历史。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight kotlin">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">int</span>'&gt; 200</span></span><br><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">requests</span>.<span class="title">structures</span>.<span class="title">CaseInsensitiveDict</span>'&gt; </span>&#123;<span class="string">'X-Runtime'</span>: <span class="string">'0.006363'</span>, <span class="string">'Connection'</span>: <span class="string">'keep-alive'</span>, <span class="string">'Content-Type'</span>: <span class="string">'text/html; charset=utf-8'</span>, <span class="string">'X-Content-Type-Options'</span>: <span class="string">'nosniff'</span>, <span class="string">'Date'</span>: <span class="string">'Sat, 27 Aug 2016 17:18:51 GMT'</span>, <span class="string">'Server'</span>: <span class="string">'nginx'</span>, <span class="string">'X-Frame-Options'</span>: <span class="string">'DENY'</span>, <span class="string">'Content-Encoding'</span>: <span class="string">'gzip'</span>, <span class="string">'Vary'</span>: <span class="string">'Accept-Encoding'</span>, <span class="string">'ETag'</span>: <span class="string">'W/"3abda885e0e123bfde06d9b61e696159"'</span>, <span class="string">'X-XSS-Protection'</span>: <span class="string">'1; mode=block'</span>, <span class="string">'X-Request-Id'</span>: <span class="string">'a8a3c4d5-f660-422f-8df9-49719dd9b5d4'</span>, <span class="string">'Transfer-Encoding'</span>: <span class="string">'chunked'</span>, <span class="string">'Set-Cookie'</span>: <span class="string">'read_mode=day; path=/, default_font=font2; path=/, _session_id=xxx; path=/; HttpOnly'</span>, <span class="string">'Cache-Control'</span>: <span class="string">'max-age=0, private, must-revalidate'</span>&#125;</span><br><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">requests</span>.<span class="title">cookies</span>.<span class="title">RequestsCookieJar</span>'&gt; &lt;<span class="type">RequestsCookieJar[&lt;Cookie _session_id=xxx for www.jianshu.com/</span>&gt;, <span class="type"></span>&lt;<span class="type">Cookie default_font=font2 for www.jianshu.com/</span>&gt;, <span class="type"></span>&lt;<span class="type">Cookie read_mode=day for www.jianshu.com/</span>&gt;]&gt;</span></span><br><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">str</span>'&gt; <span class="title">http</span>:<span class="type">//www.jianshu.com/</span></span></span><br><span class="line">&lt;<span class="class"><span class="keyword">class</span> '<span class="title">list</span>'&gt; []</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>因为<code>session_id</code>过长，在此简写。可以看到，<code>headers</code>和<code>cookies</code>这两个属性得到的结果分别是<code>CaseInsensitiveDict</code>和<code>RequestsCookieJar</code>类型。</p>
                  <p>状态码常用来判断请求是否成功，而requests还提供了一个内置的状态码查询对象<code>requests.codes</code>，示例如下：</p>
                  <figure class="highlight awk">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">import requests</span><br><span class="line"></span><br><span class="line">r = requests.get(<span class="string">'http://www.jianshu.com'</span>)</span><br><span class="line"><span class="keyword">exit</span>() <span class="keyword">if</span> not r.status_code == requests.codes.ok <span class="keyword">else</span> print(<span class="string">'Request Successfully'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里通过比较返回码和内置的成功的返回码，来保证请求得到了正常响应，输出成功请求的消息，否则程序终止，这里我们用<code>requests.codes.ok</code>得到的是成功的状态码200。</p>
                  <p>那么，肯定不能只有<code>ok</code>这个条件码。下面列出了返回码和相应的查询条件：</p>
                  <figure class="highlight less">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br><span class="line">34</span><br><span class="line">35</span><br><span class="line">36</span><br><span class="line">37</span><br><span class="line">38</span><br><span class="line">39</span><br><span class="line">40</span><br><span class="line">41</span><br><span class="line">42</span><br><span class="line">43</span><br><span class="line">44</span><br><span class="line">45</span><br><span class="line">46</span><br><span class="line">47</span><br><span class="line">48</span><br><span class="line">49</span><br><span class="line">50</span><br><span class="line">51</span><br><span class="line">52</span><br><span class="line">53</span><br><span class="line">54</span><br><span class="line">55</span><br><span class="line">56</span><br><span class="line">57</span><br><span class="line">58</span><br><span class="line">59</span><br><span class="line">60</span><br><span class="line">61</span><br><span class="line">62</span><br><span class="line">63</span><br><span class="line">64</span><br><span class="line">65</span><br><span class="line">66</span><br><span class="line">67</span><br><span class="line">68</span><br><span class="line">69</span><br><span class="line">70</span><br><span class="line">71</span><br><span class="line">72</span><br><span class="line">73</span><br><span class="line">74</span><br><span class="line">75</span><br><span class="line">76</span><br><span class="line">77</span><br><span class="line">78</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"># 信息性状态码</span><br><span class="line"><span class="selector-tag">100</span>: (<span class="string">'continue'</span>,),</span><br><span class="line"><span class="selector-tag">101</span>: (<span class="string">'switching_protocols'</span>,),</span><br><span class="line"><span class="selector-tag">102</span>: (<span class="string">'processing'</span>,),</span><br><span class="line"><span class="selector-tag">103</span>: (<span class="string">'checkpoint'</span>,),</span><br><span class="line"><span class="selector-tag">122</span>: (<span class="string">'uri_too_long'</span>, <span class="string">'request_uri_too_long'</span>),</span><br><span class="line"></span><br><span class="line"># 成功状态码</span><br><span class="line"><span class="selector-tag">200</span>: (<span class="string">'ok'</span>, <span class="string">'okay'</span>, <span class="string">'all_ok'</span>, <span class="string">'all_okay'</span>, <span class="string">'all_good'</span>, <span class="string">'\\o/'</span>, <span class="string">'✓'</span>),</span><br><span class="line"><span class="selector-tag">201</span>: (<span class="string">'created'</span>,),</span><br><span class="line"><span class="selector-tag">202</span>: (<span class="string">'accepted'</span>,),</span><br><span class="line"><span class="selector-tag">203</span>: (<span class="string">'non_authoritative_info'</span>, <span class="string">'non_authoritative_information'</span>),</span><br><span class="line"><span class="selector-tag">204</span>: (<span class="string">'no_content'</span>,),</span><br><span class="line"><span class="selector-tag">205</span>: (<span class="string">'reset_content'</span>, <span class="string">'reset'</span>),</span><br><span class="line"><span class="selector-tag">206</span>: (<span class="string">'partial_content'</span>, <span class="string">'partial'</span>),</span><br><span class="line"><span class="selector-tag">207</span>: (<span class="string">'multi_status'</span>, <span class="string">'multiple_status'</span>, <span class="string">'multi_stati'</span>, <span class="string">'multiple_stati'</span>),</span><br><span class="line"><span class="selector-tag">208</span>: (<span class="string">'already_reported'</span>,),</span><br><span class="line"><span class="selector-tag">226</span>: (<span class="string">'im_used'</span>,),</span><br><span class="line"></span><br><span class="line"># 重定向状态码</span><br><span class="line"><span class="selector-tag">300</span>: (<span class="string">'multiple_choices'</span>,),</span><br><span class="line"><span class="selector-tag">301</span>: (<span class="string">'moved_permanently'</span>, <span class="string">'moved'</span>, <span class="string">'\\o-'</span>),</span><br><span class="line"><span class="selector-tag">302</span>: (<span class="string">'found'</span>,),</span><br><span class="line"><span class="selector-tag">303</span>: (<span class="string">'see_other'</span>, <span class="string">'other'</span>),</span><br><span class="line"><span class="selector-tag">304</span>: (<span class="string">'not_modified'</span>,),</span><br><span class="line"><span class="selector-tag">305</span>: (<span class="string">'use_proxy'</span>,),</span><br><span class="line"><span class="selector-tag">306</span>: (<span class="string">'switch_proxy'</span>,),</span><br><span class="line"><span class="selector-tag">307</span>: (<span class="string">'temporary_redirect'</span>, <span class="string">'temporary_moved'</span>, <span class="string">'temporary'</span>),</span><br><span class="line"><span class="selector-tag">308</span>: (<span class="string">'permanent_redirect'</span>,</span><br><span class="line">      <span class="string">'resume_incomplete'</span>, <span class="string">'resume'</span>,), # <span class="selector-tag">These</span> <span class="selector-tag">2</span> <span class="selector-tag">to</span> <span class="selector-tag">be</span> <span class="selector-tag">removed</span> <span class="selector-tag">in</span> <span class="selector-tag">3</span><span class="selector-class">.0</span></span><br><span class="line"></span><br><span class="line"># 客户端错误状态码</span><br><span class="line"><span class="selector-tag">400</span>: (<span class="string">'bad_request'</span>, <span class="string">'bad'</span>),</span><br><span class="line"><span class="selector-tag">401</span>: (<span class="string">'unauthorized'</span>,),</span><br><span class="line"><span class="selector-tag">402</span>: (<span class="string">'payment_required'</span>, <span class="string">'payment'</span>),</span><br><span class="line"><span class="selector-tag">403</span>: (<span class="string">'forbidden'</span>,),</span><br><span class="line"><span class="selector-tag">404</span>: (<span class="string">'not_found'</span>, <span class="string">'-o-'</span>),</span><br><span class="line"><span class="selector-tag">405</span>: (<span class="string">'method_not_allowed'</span>, <span class="string">'not_allowed'</span>),</span><br><span class="line"><span class="selector-tag">406</span>: (<span class="string">'not_acceptable'</span>,),</span><br><span class="line"><span class="selector-tag">407</span>: (<span class="string">'proxy_authentication_required'</span>, <span class="string">'proxy_auth'</span>, <span class="string">'proxy_authentication'</span>),</span><br><span class="line"><span class="selector-tag">408</span>: (<span class="string">'request_timeout'</span>, <span class="string">'timeout'</span>),</span><br><span class="line"><span class="selector-tag">409</span>: (<span class="string">'conflict'</span>,),</span><br><span class="line"><span class="selector-tag">410</span>: (<span class="string">'gone'</span>,),</span><br><span class="line"><span class="selector-tag">411</span>: (<span class="string">'length_required'</span>,),</span><br><span class="line"><span class="selector-tag">412</span>: (<span class="string">'precondition_failed'</span>, <span class="string">'precondition'</span>),</span><br><span class="line"><span class="selector-tag">413</span>: (<span class="string">'request_entity_too_large'</span>,),</span><br><span class="line"><span class="selector-tag">414</span>: (<span class="string">'request_uri_too_large'</span>,),</span><br><span class="line"><span class="selector-tag">415</span>: (<span class="string">'unsupported_media_type'</span>, <span class="string">'unsupported_media'</span>, <span class="string">'media_type'</span>),</span><br><span class="line"><span class="selector-tag">416</span>: (<span class="string">'requested_range_not_satisfiable'</span>, <span class="string">'requested_range'</span>, <span class="string">'range_not_satisfiable'</span>),</span><br><span class="line"><span class="selector-tag">417</span>: (<span class="string">'expectation_failed'</span>,),</span><br><span class="line"><span class="selector-tag">418</span>: (<span class="string">'im_a_teapot'</span>, <span class="string">'teapot'</span>, <span class="string">'i_am_a_teapot'</span>),</span><br><span class="line"><span class="selector-tag">421</span>: (<span class="string">'misdirected_request'</span>,),</span><br><span class="line"><span class="selector-tag">422</span>: (<span class="string">'unprocessable_entity'</span>, <span class="string">'unprocessable'</span>),</span><br><span class="line"><span class="selector-tag">423</span>: (<span class="string">'locked'</span>,),</span><br><span class="line"><span class="selector-tag">424</span>: (<span class="string">'failed_dependency'</span>, <span class="string">'dependency'</span>),</span><br><span class="line"><span class="selector-tag">425</span>: (<span class="string">'unordered_collection'</span>, <span class="string">'unordered'</span>),</span><br><span class="line"><span class="selector-tag">426</span>: (<span class="string">'upgrade_required'</span>, <span class="string">'upgrade'</span>),</span><br><span class="line"><span class="selector-tag">428</span>: (<span class="string">'precondition_required'</span>, <span class="string">'precondition'</span>),</span><br><span class="line"><span class="selector-tag">429</span>: (<span class="string">'too_many_requests'</span>, <span class="string">'too_many'</span>),</span><br><span class="line"><span class="selector-tag">431</span>: (<span class="string">'header_fields_too_large'</span>, <span class="string">'fields_too_large'</span>),</span><br><span class="line"><span class="selector-tag">444</span>: (<span class="string">'no_response'</span>, <span class="string">'none'</span>),</span><br><span class="line"><span class="selector-tag">449</span>: (<span class="string">'retry_with'</span>, <span class="string">'retry'</span>),</span><br><span class="line"><span class="selector-tag">450</span>: (<span class="string">'blocked_by_windows_parental_controls'</span>, <span class="string">'parental_controls'</span>),</span><br><span class="line"><span class="selector-tag">451</span>: (<span class="string">'unavailable_for_legal_reasons'</span>, <span class="string">'legal_reasons'</span>),</span><br><span class="line"><span class="selector-tag">499</span>: (<span class="string">'client_closed_request'</span>,),</span><br><span class="line"></span><br><span class="line"># 服务端错误状态码</span><br><span class="line"><span class="selector-tag">500</span>: (<span class="string">'internal_server_error'</span>, <span class="string">'server_error'</span>, <span class="string">'/o\\'</span>, <span class="string">'✗'</span>),</span><br><span class="line"><span class="selector-tag">501</span>: (<span class="string">'not_implemented'</span>,),</span><br><span class="line"><span class="selector-tag">502</span>: (<span class="string">'bad_gateway'</span>,),</span><br><span class="line"><span class="selector-tag">503</span>: (<span class="string">'service_unavailable'</span>, <span class="string">'unavailable'</span>),</span><br><span class="line"><span class="selector-tag">504</span>: (<span class="string">'gateway_timeout'</span>,),</span><br><span class="line"><span class="selector-tag">505</span>: (<span class="string">'http_version_not_supported'</span>, <span class="string">'http_version'</span>),</span><br><span class="line"><span class="selector-tag">506</span>: (<span class="string">'variant_also_negotiates'</span>,),</span><br><span class="line"><span class="selector-tag">507</span>: (<span class="string">'insufficient_storage'</span>,),</span><br><span class="line"><span class="selector-tag">509</span>: (<span class="string">'bandwidth_limit_exceeded'</span>, <span class="string">'bandwidth'</span>),</span><br><span class="line"><span class="selector-tag">510</span>: (<span class="string">'not_extended'</span>,),</span><br><span class="line"><span class="selector-tag">511</span>: (<span class="string">'network_authentication_required'</span>, <span class="string">'network_auth'</span>, <span class="string">'network_authentication'</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>比如，如果想判断结果是不是404状态，可以用<code>requests.codes.not_found</code>来比对。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-27 14:14:47" itemprop="dateCreated datePublished" datetime="2018-01-27T14:14:47+08:00">2018-01-27</time>
                </span>
                <span id="/5517.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 3.2.1-基本用法" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>11k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>10 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5514.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5514.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 3.2-使用requests</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>上一节中，我们了解了urllib的基本用法，但是其中确实有不方便的地方，比如处理网页验证和Cookies时，需要写<code>Opener</code>和<code>Handler</code>来处理。为了更加方便地实现这些操作，就有了更为强大的库requests，有了它，Cookies、登录验证、代理设置等操作都不是事儿。</p>
                  <p>接下来，让我们领略一下它的强大之处吧。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-27 14:12:28" itemprop="dateCreated datePublished" datetime="2018-01-27T14:12:28+08:00">2018-01-27</time>
                </span>
                <span id="/5514.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 3.2-使用requests" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>156</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>1 分钟</span>
                </span>
              </div>
            </article>
            <article itemscope itemtype="http://schema.org/Article" class="post-block index" lang="zh-CN">
              <link itemprop="mainEntityOfPage" href="https://cuiqingcai.com/5511.html">
              <span hidden itemprop="author" itemscope itemtype="http://schema.org/Person">
                <meta itemprop="image" content="/images/avatar.png">
                <meta itemprop="name" content="崔庆才">
                <meta itemprop="description" content="崔庆才的个人站点，记录生活的瞬间，分享学习的心得。">
              </span>
              <span hidden itemprop="publisher" itemscope itemtype="http://schema.org/Organization">
                <meta itemprop="name" content="静觅">
              </span>
              <header class="post-header">
                <h2 class="post-title" itemprop="name headline">
                  <a class="label"> Python <i class="label-arrow"></i>
                  </a>
                  <a href="/5511.html" class="post-title-link" itemprop="url">[Python3网络爬虫开发实战] 3.1.4-分析Robots协议</a>
                </h2>
              </header>
              <div class="post-body" itemprop="articleBody">
                <div class="thumb">
                  <img itemprop="contentUrl" class="random">
                </div>
                <div class="excerpt">
                  <p>
                  <p>利用urllib的<code>robotparser</code>模块，我们可以实现网站Robots协议的分析。本节中，我们来简单了解一下该模块的用法。</p>
                  <h2 id="1-Robots协议"><a href="#1-Robots协议" class="headerlink" title="1. Robots协议"></a>1. Robots协议</h2>
                  <p>Robots协议也称作爬虫协议、机器人协议，它的全名叫作网络爬虫排除标准（Robots Exclusion Protocol），用来告诉爬虫和搜索引擎哪些页面可以抓取，哪些不可以抓取。它通常是一个叫作robots.txt的文本文件，一般放在网站的根目录下。</p>
                  <p>当搜索爬虫访问一个站点时，它首先会检查这个站点根目录下是否存在robots.txt文件，如果存在，搜索爬虫会根据其中定义的爬取范围来爬取。如果没有找到这个文件，搜索爬虫便会访问所有可直接访问的页面。</p>
                  <p>下面我们看一个robots.txt的样例：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">User-<span class="string">agent:</span> *</span><br><span class="line"><span class="string">Disallow:</span> /</span><br><span class="line"><span class="string">Allow:</span> <span class="regexp">/public/</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这实现了对所有搜索爬虫只允许爬取public目录的功能，将上述内容保存成robots.txt文件，放在网站的根目录下，和网站的入口文件（比如index.php、index.html和index.jsp等）放在一起。</p>
                  <p>上面的<code>User-agent</code>描述了搜索爬虫的名称，这里将其设置为*则代表该协议对任何爬取爬虫有效。比如，我们可以设置：</p>
                  <figure class="highlight dockerfile">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="keyword">User</span>-agent: Baiduspider</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这就代表我们设置的规则对百度爬虫是有效的。如果有多条<code>User-agent</code>记录，则就会有多个爬虫会受到爬取限制，但至少需要指定一条。</p>
                  <p><code>Disallow</code>指定了不允许抓取的目录，比如上例子中设置为/则代表不允许抓取所有页面。</p>
                  <p><code>Allow</code>一般和<code>Disallow</code>一起使用，一般不会单独使用，用来排除某些限制。现在我们设置为<code>/public/</code>，则表示所有页面不允许抓取，但可以抓取public目录。</p>
                  <p>下面我们再来看几个例子。禁止所有爬虫访问任何目录的代码如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">User-<span class="string">agent:</span> * </span><br><span class="line"><span class="string">Disallow:</span> /</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>允许所有爬虫访问任何目录的代码如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">User-<span class="string">agent:</span> *</span><br><span class="line"><span class="string">Disallow:</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>另外，直接把robots.txt文件留空也是可以的。</p>
                  <p>禁止所有爬虫访问网站某些目录的代码如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">User-<span class="string">agent:</span> *</span><br><span class="line"><span class="string">Disallow:</span> <span class="regexp">/private/</span></span><br><span class="line"><span class="string">Disallow:</span> <span class="regexp">/tmp/</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>只允许某一个爬虫访问的代码如下：</p>
                  <figure class="highlight groovy">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">User-<span class="string">agent:</span> WebCrawler</span><br><span class="line"><span class="string">Disallow:</span></span><br><span class="line">User-<span class="string">agent:</span> *</span><br><span class="line"><span class="string">Disallow:</span> /</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这些是robots.txt的一些常见写法。</p>
                  <h2 id="2-爬虫名称"><a href="#2-爬虫名称" class="headerlink" title="2. 爬虫名称"></a>2. 爬虫名称</h2>
                  <p>大家可能会疑惑，爬虫名是哪儿来的？为什么就叫这个名？其实它是有固定名字的了，比如百度的就叫作BaiduSpider。表3-1列出了一些常见的搜索爬虫的名称及对应的网站。</p>
                  <p>表3-1 一些常见搜索爬虫的名称及其对应的网站</p>
                  <p>爬虫名称</p>
                  <p>名称</p>
                  <p>网站</p>
                  <p>BaiduSpider</p>
                  <p>百度</p>
                  <p>www.baidu.com</p>
                  <p>Googlebot</p>
                  <p>谷歌</p>
                  <p>www.google.com</p>
                  <p>360Spider</p>
                  <p>360搜索</p>
                  <p>www.so.com</p>
                  <p>YodaoBot</p>
                  <p>有道</p>
                  <p>www.youdao.com</p>
                  <p>ia_archiver</p>
                  <p>Alexa</p>
                  <p>www.alexa.cn</p>
                  <p>Scooter</p>
                  <p>altavista</p>
                  <p>www.altavista.com</p>
                  <h2 id="3-robotparser"><a href="#3-robotparser" class="headerlink" title="3. robotparser"></a>3. robotparser</h2>
                  <p>了解Robots协议之后，我们就可以使用<code>robotparser</code>模块来解析robots.txt了。该模块提供了一个类<code>RobotFileParser</code>，它可以根据某网站的robots.txt文件来判断一个爬取爬虫是否有权限来爬取这个网页。</p>
                  <p>该类用起来非常简单，只需要在构造方法里传入robots.txt的链接即可。首先看一下它的声明：</p>
                  <figure class="highlight routeros">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">urllib.robotparser.RobotFileParser(<span class="attribute">url</span>=<span class="string">''</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>当然，也可以在声明时不传入，默认为空，最后再使用<code>set_url()</code>方法设置一下也可。</p>
                  <p>下面列出了这个类常用的几个方法。</p>
                  <ul>
                    <li><strong><code>set_url()</code></strong>：用来设置robots.txt文件的链接。如果在创建<code>RobotFileParser</code>对象时传入了链接，那么就不需要再使用这个方法设置了。</li>
                    <li><strong><code>read()</code></strong>：读取robots.txt文件并进行分析。注意，这个方法执行一个读取和分析操作，如果不调用这个方法，接下来的判断都会为<code>False</code>，所以一定记得调用这个方法。这个方法不会返回任何内容，但是执行了读取操作。</li>
                    <li><strong><code>parse()</code></strong>：用来解析robots.txt文件，传入的参数是robots.txt某些行的内容，它会按照robots.txt的语法规则来分析这些内容。</li>
                    <li><strong><code>can_fetch()</code></strong>：该方法传入两个参数，第一个是<code>User-agent</code>，第二个是要抓取的URL。返回的内容是该搜索引擎是否可以抓取这个URL，返回结果是<code>True</code>或<code>False</code>。</li>
                    <li><strong><code>mtime()</code></strong>：返回的是上次抓取和分析robots.txt的时间，这对于长时间分析和抓取的搜索爬虫是很有必要的，你可能需要定期检查来抓取最新的robots.txt。</li>
                    <li><strong><code>modified()</code></strong>：它同样对长时间分析和抓取的搜索爬虫很有帮助，将当前时间设置为上次抓取和分析robots.txt的时间。</li>
                  </ul>
                  <p>下面我们用实例来看一下：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from urllib.robotparser import RobotFileParser</span><br><span class="line"></span><br><span class="line">rp = <span class="constructor">RobotFileParser()</span></span><br><span class="line">rp.set<span class="constructor">_url('<span class="params">http</span>:<span class="operator">/</span><span class="operator">/</span><span class="params">www</span>.<span class="params">jianshu</span>.<span class="params">com</span><span class="operator">/</span><span class="params">robots</span>.<span class="params">txt</span>')</span></span><br><span class="line">rp.read<span class="literal">()</span></span><br><span class="line">print(rp.can<span class="constructor">_fetch('<span class="operator">*</span>', '<span class="params">http</span>:<span class="operator">/</span><span class="operator">/</span><span class="params">www</span>.<span class="params">jianshu</span>.<span class="params">com</span><span class="operator">/</span><span class="params">p</span><span class="operator">/</span><span class="params">b67554025d7d</span>')</span>)</span><br><span class="line">print(rp.can<span class="constructor">_fetch('<span class="operator">*</span>', <span class="string">"http://www.jianshu.com/search?q=python&amp;page=1&amp;type=collections"</span>)</span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里以简书为例，首先创建<code>RobotFileParser</code>对象，然后通过<code>set_url()</code>方法设置了robots.txt的链接。当然，不用这个方法的话，可以在声明时直接用如下方法设置：</p>
                  <figure class="highlight reasonml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">rp = <span class="constructor">RobotFileParser('<span class="params">http</span>:<span class="operator">/</span><span class="operator">/</span><span class="params">www</span>.<span class="params">jianshu</span>.<span class="params">com</span><span class="operator">/</span><span class="params">robots</span>.<span class="params">txt</span>')</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>接着利用<code>can_fetch()</code>方法判断了网页是否可以被抓取。</p>
                  <p>运行结果如下：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="literal">True</span></span><br><span class="line"><span class="literal">False</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>这里同样可以使用<code>parse()</code>方法执行读取和分析，示例如下：</p>
                  <figure class="highlight stylus">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line">from urllib<span class="selector-class">.robotparser</span> import RobotFileParser</span><br><span class="line">from urllib<span class="selector-class">.request</span> import urlopen</span><br><span class="line"></span><br><span class="line">rp = RobotFileParser()</span><br><span class="line">rp.parse(urlopen(<span class="string">'http://www.jianshu.com/robots.txt'</span>).read().decode(<span class="string">'utf-8'</span>).split(<span class="string">'\n'</span>))</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(rp.can_fetch(<span class="string">'*'</span>, <span class="string">'http://www.jianshu.com/p/b67554025d7d'</span>)</span></span>)</span><br><span class="line"><span class="function"><span class="title">print</span><span class="params">(rp.can_fetch(<span class="string">'*'</span>, <span class="string">"http://www.jianshu.com/search?q=python&amp;page=1&amp;type=collections"</span>)</span></span>)</span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>运行结果一样：</p>
                  <figure class="highlight yaml">
                    <table>
                      <tr>
                        <td class="gutter">
                          <pre><span class="line">1</span><br><span class="line">2</span><br></pre>
                        </td>
                        <td class="code">
                          <pre><span class="line"><span class="literal">True</span></span><br><span class="line"><span class="literal">False</span></span><br></pre>
                        </td>
                      </tr>
                    </table>
                  </figure>
                  <p>本节介绍了<code>robotparser</code>模块的基本用法和实例，利用它，我们可以方便地判断哪些页面可以抓取，哪些页面不可以抓取。</p>
                  </p>
                </div>
              </div>
              <div class="post-meta">
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-user"></i>
                  </span>
                  <span class="post-meta-item-text">作者</span>
                  <span><a href="/authors/崔庆才" class="author" itemprop="url" rel="index">崔庆才</a></span>
                </span>
                <span class="post-meta-item">
                  <span class="post-meta-item-icon">
                    <i class="far fa-calendar"></i>
                  </span>
                  <span class="post-meta-item-text">发表于</span>
                  <time title="创建时间：2018-01-27 14:09:21" itemprop="dateCreated datePublished" datetime="2018-01-27T14:09:21+08:00">2018-01-27</time>
                </span>
                <span id="/5511.html" class="post-meta-item leancloud_visitors" data-flag-title="[Python3网络爬虫开发实战] 3.1.4-分析Robots协议" title="阅读次数">
                  <span class="post-meta-item-icon">
                    <i class="fa fa-eye"></i>
                  </span>
                  <span class="post-meta-item-text">阅读次数：</span>
                  <span class="leancloud-visitors-count"></span>
                </span>
                <span class="post-meta-item" title="本文字数">
                  <span class="post-meta-item-icon">
                    <i class="far fa-file-word"></i>
                  </span>
                  <span class="post-meta-item-text">本文字数：</span>
                  <span>3k</span>
                </span>
                <span class="post-meta-item" title="阅读时长">
                  <span class="post-meta-item-icon">
                    <i class="far fa-clock"></i>
                  </span>
                  <span class="post-meta-item-text">阅读时长 &asymp;</span>
                  <span>3 分钟</span>
                </span>
              </div>
            </article>
            <script>
              document.querySelectorAll('.random').forEach(item => item.src = "https://picsum.photos/id/" + Math.floor(Math.random() * Math.floor(300)) + "/200/133")

            </script>
            <nav class="pagination">
              <a class="extend prev" rel="prev" href="/page/16/"><i class="fa fa-angle-left" aria-label="上一页"></i></a><a class="page-number" href="/">1</a><span class="space">&hellip;</span><a class="page-number" href="/page/16/">16</a><span class="page-number current">17</span><a class="page-number" href="/page/18/">18</a><span class="space">&hellip;</span><a class="page-number" href="/page/31/">31</a><a class="extend next" rel="next" href="/page/18/"><i class="fa fa-angle-right" aria-label="下一页"></i></a>
            </nav>
          </div>
          <script>
            window.addEventListener('tabs:register', () =>
            {
              let
              {
                activeClass
              } = CONFIG.comments;
              if (CONFIG.comments.storage)
              {
                activeClass = localStorage.getItem('comments_active') || activeClass;
              }
              if (activeClass)
              {
                let activeTab = document.querySelector(`a[href="#comment-${activeClass}"]`);
                if (activeTab)
                {
                  activeTab.click();
                }
              }
            });
            if (CONFIG.comments.storage)
            {
              window.addEventListener('tabs:click', event =>
              {
                if (!event.target.matches('.tabs-comment .tab-content .tab-pane')) return;
                let commentClass = event.target.classList[1];
                localStorage.setItem('comments_active', commentClass);
              });
            }

          </script>
        </div>
        <div class="toggle sidebar-toggle">
          <span class="toggle-line toggle-line-first"></span>
          <span class="toggle-line toggle-line-middle"></span>
          <span class="toggle-line toggle-line-last"></span>
        </div>
        <aside class="sidebar">
          <div class="sidebar-inner">
            <ul class="sidebar-nav motion-element">
              <li class="sidebar-nav-toc"> 文章目录 </li>
              <li class="sidebar-nav-overview"> 站点概览 </li>
            </ul>
            <!--noindex-->
            <div class="post-toc-wrap sidebar-panel">
            </div>
            <!--/noindex-->
            <div class="site-overview-wrap sidebar-panel">
              <div class="site-author motion-element" itemprop="author" itemscope itemtype="http://schema.org/Person">
                <img class="site-author-image" itemprop="image" alt="崔庆才" src="/images/avatar.png">
                <p class="site-author-name" itemprop="name">崔庆才</p>
                <div class="site-description" itemprop="description">崔庆才的个人站点，记录生活的瞬间，分享学习的心得。</div>
              </div>
              <div class="site-state-wrap motion-element">
                <nav class="site-state">
                  <div class="site-state-item site-state-posts">
                    <a href="/archives/">
                      <span class="site-state-item-count">608</span>
                      <span class="site-state-item-name">日志</span>
                    </a>
                  </div>
                  <div class="site-state-item site-state-categories">
                    <a href="/categories/">
                      <span class="site-state-item-count">24</span>
                      <span class="site-state-item-name">分类</span></a>
                  </div>
                  <div class="site-state-item site-state-tags">
                    <a href="/tags/">
                      <span class="site-state-item-count">156</span>
                      <span class="site-state-item-name">标签</span></a>
                  </div>
                </nav>
              </div>
              <div class="links-of-author motion-element">
                <span class="links-of-author-item">
                  <a href="https://github.com/Germey" title="GitHub → https:&#x2F;&#x2F;github.com&#x2F;Germey" rel="noopener" target="_blank"><i class="fab fa-github fa-fw"></i>GitHub</a>
                </span>
                <span class="links-of-author-item">
                  <a href="mailto:cqc@cuiqingcai.com.com" title="邮件 → mailto:cqc@cuiqingcai.com.com" rel="noopener" target="_blank"><i class="fa fa-envelope fa-fw"></i>邮件</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://weibo.com/cuiqingcai" title="微博 → https:&#x2F;&#x2F;weibo.com&#x2F;cuiqingcai" rel="noopener" target="_blank"><i class="fab fa-weibo fa-fw"></i>微博</a>
                </span>
                <span class="links-of-author-item">
                  <a href="https://www.zhihu.com/people/Germey" title="知乎 → https:&#x2F;&#x2F;www.zhihu.com&#x2F;people&#x2F;Germey" rel="noopener" target="_blank"><i class="fa fa-magic fa-fw"></i>知乎</a>
                </span>
              </div>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="https://tutorial.lengyue.video/?coupon=12ef4b1a-a3db-11ea-bb37-0242ac130002_cqx_850" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/bco2a.png" style=" width: 100%;">
              </a>
            </div>
            <div style=" width: 100%;" class="sidebar-panel sidebar-panel-image sidebar-panel-active">
              <a href="http://www.ipidea.net/?utm-source=cqc&utm-keyword=?cqc" target="_blank" rel="noopener">
                <img src="https://qiniu.cuiqingcai.com/0ywun.png" style=" width: 100%;">
              </a>
            </div>
            <div class="sidebar-panel sidebar-panel-tags sidebar-panel-active">
              <h4 class="name"> 标签云 </h4>
              <div class="content">
                <a href="/tags/2048/" style="font-size: 10px;">2048</a> <a href="/tags/API/" style="font-size: 10px;">API</a> <a href="/tags/Bootstrap/" style="font-size: 11.25px;">Bootstrap</a> <a href="/tags/CDN/" style="font-size: 10px;">CDN</a> <a href="/tags/CQC/" style="font-size: 10px;">CQC</a> <a href="/tags/CSS/" style="font-size: 10px;">CSS</a> <a href="/tags/CSS-%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">CSS 反爬虫</a> <a href="/tags/CV/" style="font-size: 10px;">CV</a> <a href="/tags/Django/" style="font-size: 10px;">Django</a> <a href="/tags/Eclipse/" style="font-size: 11.25px;">Eclipse</a> <a href="/tags/FTP/" style="font-size: 10px;">FTP</a> <a href="/tags/Git/" style="font-size: 10px;">Git</a> <a href="/tags/GitHub/" style="font-size: 13.75px;">GitHub</a> <a href="/tags/HTML5/" style="font-size: 10px;">HTML5</a> <a href="/tags/Hexo/" style="font-size: 10px;">Hexo</a> <a href="/tags/IT/" style="font-size: 10px;">IT</a> <a href="/tags/JSP/" style="font-size: 10px;">JSP</a> <a href="/tags/JavaScript/" style="font-size: 10px;">JavaScript</a> <a href="/tags/K8s/" style="font-size: 10px;">K8s</a> <a href="/tags/LOGO/" style="font-size: 10px;">LOGO</a> <a href="/tags/Linux/" style="font-size: 10px;">Linux</a> <a href="/tags/MIUI/" style="font-size: 10px;">MIUI</a> <a href="/tags/MongoDB/" style="font-size: 10px;">MongoDB</a> <a href="/tags/Mysql/" style="font-size: 10px;">Mysql</a> <a href="/tags/NBA/" style="font-size: 10px;">NBA</a> <a href="/tags/PHP/" style="font-size: 11.25px;">PHP</a> <a href="/tags/PS/" style="font-size: 10px;">PS</a> <a href="/tags/Pathlib/" style="font-size: 10px;">Pathlib</a> <a href="/tags/PhantomJS/" style="font-size: 10px;">PhantomJS</a> <a href="/tags/Python/" style="font-size: 15px;">Python</a> <a href="/tags/Python3/" style="font-size: 12.5px;">Python3</a> <a href="/tags/Pythonic/" style="font-size: 10px;">Pythonic</a> <a href="/tags/QQ/" style="font-size: 10px;">QQ</a> <a href="/tags/Redis/" style="font-size: 10px;">Redis</a> <a href="/tags/SAE/" style="font-size: 10px;">SAE</a> <a href="/tags/SSH/" style="font-size: 10px;">SSH</a> <a href="/tags/SVG/" style="font-size: 10px;">SVG</a> <a href="/tags/Scrapy/" style="font-size: 10px;">Scrapy</a> <a href="/tags/Scrapy-redis/" style="font-size: 10px;">Scrapy-redis</a> <a href="/tags/Scrapy%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">Scrapy分布式</a> <a href="/tags/Selenium/" style="font-size: 10px;">Selenium</a> <a href="/tags/TKE/" style="font-size: 10px;">TKE</a> <a href="/tags/Ubuntu/" style="font-size: 11.25px;">Ubuntu</a> <a href="/tags/VS-Code/" style="font-size: 10px;">VS Code</a> <a href="/tags/Vs-Code/" style="font-size: 10px;">Vs Code</a> <a href="/tags/Vue/" style="font-size: 11.25px;">Vue</a> <a href="/tags/Webpack/" style="font-size: 10px;">Webpack</a> <a href="/tags/Windows/" style="font-size: 10px;">Windows</a> <a href="/tags/Winpcap/" style="font-size: 10px;">Winpcap</a> <a href="/tags/WordPress/" style="font-size: 13.75px;">WordPress</a> <a href="/tags/Youtube/" style="font-size: 11.25px;">Youtube</a> <a href="/tags/android/" style="font-size: 10px;">android</a> <a href="/tags/ansible/" style="font-size: 10px;">ansible</a> <a href="/tags/cocos2d-x/" style="font-size: 10px;">cocos2d-x</a> <a href="/tags/e6/" style="font-size: 10px;">e6</a> <a href="/tags/fitvids/" style="font-size: 10px;">fitvids</a> <a href="/tags/git/" style="font-size: 11.25px;">git</a> <a href="/tags/json/" style="font-size: 10px;">json</a> <a href="/tags/js%E9%80%86%E5%90%91/" style="font-size: 10px;">js逆向</a> <a href="/tags/kubernetes/" style="font-size: 10px;">kubernetes</a> <a href="/tags/log/" style="font-size: 10px;">log</a> <a href="/tags/logging/" style="font-size: 10px;">logging</a> <a href="/tags/matlab/" style="font-size: 11.25px;">matlab</a> <a href="/tags/python/" style="font-size: 20px;">python</a> <a href="/tags/pytube/" style="font-size: 11.25px;">pytube</a> <a href="/tags/pywin32/" style="font-size: 10px;">pywin32</a> <a href="/tags/style/" style="font-size: 10px;">style</a> <a href="/tags/tomcat/" style="font-size: 10px;">tomcat</a> <a href="/tags/ubuntu/" style="font-size: 10px;">ubuntu</a> <a href="/tags/uwsgi/" style="font-size: 10px;">uwsgi</a> <a href="/tags/vsftpd/" style="font-size: 10px;">vsftpd</a> <a href="/tags/wamp/" style="font-size: 10px;">wamp</a> <a href="/tags/wineQQ/" style="font-size: 10px;">wineQQ</a> <a href="/tags/%E4%B8%83%E7%89%9B/" style="font-size: 11.25px;">七牛</a> <a href="/tags/%E4%B8%8A%E6%B5%B7/" style="font-size: 10px;">上海</a> <a href="/tags/%E4%B8%AA%E4%BA%BA%E7%BD%91%E7%AB%99/" style="font-size: 10px;">个人网站</a> <a href="/tags/%E4%B8%BB%E9%A2%98/" style="font-size: 10px;">主题</a> <a href="/tags/%E4%BA%91%E4%BA%A7%E5%93%81/" style="font-size: 10px;">云产品</a> <a href="/tags/%E4%BA%91%E5%AD%98%E5%82%A8/" style="font-size: 10px;">云存储</a> <a href="/tags/%E4%BA%AC%E4%B8%9C%E4%BA%91/" style="font-size: 10px;">京东云</a> <a href="/tags/%E4%BA%BA%E5%B7%A5%E6%99%BA%E8%83%BD/" style="font-size: 12.5px;">人工智能</a> <a href="/tags/%E4%BB%A3%E7%90%86/" style="font-size: 10px;">代理</a> <a href="/tags/%E4%BB%A3%E7%A0%81/" style="font-size: 10px;">代码</a> <a href="/tags/%E4%BB%A3%E7%A0%81%E5%88%86%E4%BA%AB%E5%9B%BE/" style="font-size: 10px;">代码分享图</a> <a href="/tags/%E4%BC%98%E5%8C%96/" style="font-size: 10px;">优化</a> <a href="/tags/%E4%BD%8D%E8%BF%90%E7%AE%97/" style="font-size: 10px;">位运算</a> <a href="/tags/%E5%85%AC%E4%BC%97%E5%8F%B7/" style="font-size: 10px;">公众号</a> <a href="/tags/%E5%88%86%E4%BA%AB/" style="font-size: 10px;">分享</a> <a href="/tags/%E5%88%86%E5%B8%83%E5%BC%8F/" style="font-size: 10px;">分布式</a> <a href="/tags/%E5%88%9B%E4%B8%9A/" style="font-size: 10px;">创业</a> <a href="/tags/%E5%89%8D%E7%AB%AF/" style="font-size: 12.5px;">前端</a> <a href="/tags/%E5%8D%9A%E5%AE%A2/" style="font-size: 10px;">博客</a> <a href="/tags/%E5%8E%9F%E7%94%9FAPP/" style="font-size: 10px;">原生APP</a> <a href="/tags/%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 12.5px;">反爬虫</a> <a href="/tags/%E5%91%BD%E4%BB%A4/" style="font-size: 10px;">命令</a> <a href="/tags/%E5%93%8D%E5%BA%94%E5%BC%8F%E5%B8%83%E5%B1%80/" style="font-size: 10px;">响应式布局</a> <a href="/tags/%E5%9E%83%E5%9C%BE%E9%82%AE%E4%BB%B6/" style="font-size: 10px;">垃圾邮件</a> <a href="/tags/%E5%9F%9F%E5%90%8D%E7%BB%91%E5%AE%9A/" style="font-size: 10px;">域名绑定</a> <a href="/tags/%E5%A4%8D%E7%9B%98/" style="font-size: 10px;">复盘</a> <a href="/tags/%E5%A4%A7%E4%BC%97%E7%82%B9%E8%AF%84/" style="font-size: 10px;">大众点评</a> <a href="/tags/%E5%AD%97%E4%BD%93%E5%8F%8D%E7%88%AC%E8%99%AB/" style="font-size: 10px;">字体反爬虫</a> <a href="/tags/%E5%AD%97%E7%AC%A6%E9%97%AE%E9%A2%98/" style="font-size: 10px;">字符问题</a> <a href="/tags/%E5%AD%A6%E4%B9%A0%E6%96%B9%E6%B3%95/" style="font-size: 10px;">学习方法</a> <a href="/tags/%E5%AE%89%E5%8D%93/" style="font-size: 10px;">安卓</a> <a href="/tags/%E5%AE%9E%E7%94%A8/" style="font-size: 10px;">实用</a> <a href="/tags/%E5%B0%81%E9%9D%A2/" style="font-size: 10px;">封面</a> <a href="/tags/%E5%B4%94%E5%BA%86%E6%89%8D/" style="font-size: 18.75px;">崔庆才</a> <a href="/tags/%E5%B7%A5%E5%85%B7/" style="font-size: 12.5px;">工具</a> <a href="/tags/%E5%BC%80%E5%8F%91%E5%B7%A5%E5%85%B7/" style="font-size: 10px;">开发工具</a> <a href="/tags/%E5%BE%AE%E8%BD%AF/" style="font-size: 10px;">微软</a> <a href="/tags/%E6%80%9D%E8%80%83/" style="font-size: 10px;">思考</a> <a href="/tags/%E6%89%8B%E6%9C%BA%E8%AE%BF%E9%97%AE/" style="font-size: 10px;">手机访问</a> <a href="/tags/%E6%95%99%E7%A8%8B/" style="font-size: 10px;">教程</a> <a href="/tags/%E6%95%99%E8%82%B2/" style="font-size: 10px;">教育</a> <a href="/tags/%E6%96%B0%E4%B9%A6/" style="font-size: 12.5px;">新书</a> <a href="/tags/%E6%96%B9%E6%B3%95%E8%AE%BA/" style="font-size: 10px;">方法论</a> <a href="/tags/%E6%97%85%E6%B8%B8/" style="font-size: 10px;">旅游</a> <a href="/tags/%E6%97%A5%E5%BF%97/" style="font-size: 10px;">日志</a> <a href="/tags/%E6%9A%97%E6%97%B6%E9%97%B4/" style="font-size: 10px;">暗时间</a> <a href="/tags/%E6%9D%9C%E5%85%B0%E7%89%B9/" style="font-size: 11.25px;">杜兰特</a> <a href="/tags/%E6%A1%8C%E9%9D%A2/" style="font-size: 10px;">桌面</a> <a href="/tags/%E6%AD%8C%E5%8D%95/" style="font-size: 10px;">歌单</a> <a href="/tags/%E6%B1%9F%E5%8D%97/" style="font-size: 10px;">江南</a> <a href="/tags/%E6%B8%B8%E6%88%8F/" style="font-size: 10px;">游戏</a> <a href="/tags/%E7%84%A6%E8%99%91/" style="font-size: 10px;">焦虑</a> <a href="/tags/%E7%88%AC%E8%99%AB/" style="font-size: 16.25px;">爬虫</a> <a href="/tags/%E7%88%AC%E8%99%AB%E4%B9%A6%E7%B1%8D/" style="font-size: 11.25px;">爬虫书籍</a> <a href="/tags/%E7%8E%AF%E5%A2%83%E5%8F%98%E9%87%8F/" style="font-size: 10px;">环境变量</a> <a href="/tags/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/" style="font-size: 10px;">生活笔记</a> <a href="/tags/%E7%99%BB%E5%BD%95/" style="font-size: 10px;">登录</a> <a href="/tags/%E7%9F%A5%E4%B9%8E/" style="font-size: 10px;">知乎</a> <a href="/tags/%E7%9F%AD%E4%BF%A1/" style="font-size: 10px;">短信</a> <a href="/tags/%E7%9F%AD%E4%BF%A1%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">短信验证码</a> <a href="/tags/%E7%AC%94%E8%AE%B0%E8%BD%AF%E4%BB%B6/" style="font-size: 10px;">笔记软件</a> <a href="/tags/%E7%AF%AE%E7%BD%91/" style="font-size: 10px;">篮网</a> <a href="/tags/%E7%BA%B8%E5%BC%A0/" style="font-size: 10px;">纸张</a> <a href="/tags/%E7%BB%84%E4%BB%B6/" style="font-size: 10px;">组件</a> <a href="/tags/%E7%BD%91%E7%AB%99/" style="font-size: 10px;">网站</a> <a href="/tags/%E7%BD%91%E7%BB%9C%E7%88%AC%E8%99%AB/" style="font-size: 11.25px;">网络爬虫</a> <a href="/tags/%E7%BE%8E%E5%AD%A6/" style="font-size: 10px;">美学</a> <a href="/tags/%E8%82%89%E5%A4%B9%E9%A6%8D/" style="font-size: 10px;">肉夹馍</a> <a href="/tags/%E8%85%BE%E8%AE%AF%E4%BA%91/" style="font-size: 10px;">腾讯云</a> <a href="/tags/%E8%87%AA%E5%BE%8B/" style="font-size: 10px;">自律</a> <a href="/tags/%E8%A5%BF%E5%B0%91%E7%88%B7/" style="font-size: 10px;">西少爷</a> <a href="/tags/%E8%A7%86%E9%A2%91/" style="font-size: 10px;">视频</a> <a href="/tags/%E8%B0%B7%E6%AD%8C%E9%AA%8C%E8%AF%81%E7%A0%81/" style="font-size: 10px;">谷歌验证码</a> <a href="/tags/%E8%BF%90%E8%90%A5/" style="font-size: 10px;">运营</a> <a href="/tags/%E8%BF%9C%E7%A8%8B/" style="font-size: 10px;">远程</a> <a href="/tags/%E9%80%86%E5%90%91/" style="font-size: 10px;">逆向</a> <a href="/tags/%E9%85%8D%E7%BD%AE/" style="font-size: 10px;">配置</a> <a href="/tags/%E9%87%8D%E8%A3%85/" style="font-size: 10px;">重装</a> <a href="/tags/%E9%98%BF%E6%9D%9C/" style="font-size: 10px;">阿杜</a> <a href="/tags/%E9%9D%99%E8%A7%85/" style="font-size: 17.5px;">静觅</a> <a href="/tags/%E9%A2%A0%E8%A6%86/" style="font-size: 10px;">颠覆</a> <a href="/tags/%E9%A3%9E%E4%BF%A1/" style="font-size: 10px;">飞信</a> <a href="/tags/%E9%B8%BF%E8%92%99/" style="font-size: 10px;">鸿蒙</a>
              </div>
              <script>
                const tagsColors = ['#00a67c', '#5cb85c', '#d9534f', '#567e95', '#b37333', '#f4843d', '#15a287']
                const tagsElements = document.querySelectorAll('.sidebar-panel-tags .content a')
                tagsElements.forEach((item) =>
                {
                  item.style.backgroundColor = tagsColors[Math.floor(Math.random() * tagsColors.length)]
                })

              </script>
            </div>
            <div class="sidebar-panel sidebar-panel-categories sidebar-panel-active">
              <h4 class="name"> 分类 </h4>
              <div class="content">
                <ul class="category-list">
                  <li class="category-list-item"><a class="category-list-link" href="/categories/C-C/">C/C++</a><span class="category-list-count">23</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/HTML/">HTML</a><span class="category-list-count">14</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Java/">Java</a><span class="category-list-count">5</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/JavaScript/">JavaScript</a><span class="category-list-count">26</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Linux/">Linux</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Markdown/">Markdown</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Net/">Net</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Other/">Other</a><span class="category-list-count">39</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/PHP/">PHP</a><span class="category-list-count">27</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Paper/">Paper</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/Python/">Python</a><span class="category-list-count">261</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/TypeScript/">TypeScript</a><span class="category-list-count">2</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E5%B1%95%E7%A4%BA/">个人展示</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E6%97%A5%E8%AE%B0/">个人日记</a><span class="category-list-count">9</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E8%AE%B0%E5%BD%95/">个人记录</a><span class="category-list-count">4</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E4%B8%AA%E4%BA%BA%E9%9A%8F%E7%AC%94/">个人随笔</a><span class="category-list-count">15</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E5%AE%89%E8%A3%85%E9%85%8D%E7%BD%AE/">安装配置</a><span class="category-list-count">59</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%8A%80%E6%9C%AF%E6%9D%82%E8%B0%88/">技术杂谈</a><span class="category-list-count">88</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E6%9C%AA%E5%88%86%E7%B1%BB/">未分类</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%94%9F%E6%B4%BB%E7%AC%94%E8%AE%B0/">生活笔记</a><span class="category-list-count">1</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E7%A6%8F%E5%88%A9%E4%B8%93%E5%8C%BA/">福利专区</a><span class="category-list-count">6</span></li>
                  <li class="category-list-item"><a class="category-list-link" href="/categories/%E8%81%8C%E4%BD%8D%E6%8E%A8%E8%8D%90/">职位推荐</a><span class="category-list-count">2</span></li>
                </ul>
              </div>
            </div>
            <div class="sidebar-panel sidebar-panel-friends sidebar-panel-active">
              <h4 class="name"> 友情链接 </h4>
              <ul class="friends">
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/j2dub.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.findhao.net/" target="_blank" rel="noopener">FindHao</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ou6mm.jpg">
                  </span>
                  <span class="link">
                    <a href="https://diygod.me/" target="_blank" rel="noopener">DIYgod</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/6apxu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.51dev.com/" target="_blank" rel="noopener">IT技术社区</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.jankl.com/img/titleshu.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.jankl.com/" target="_blank" rel="noopener">liberalist</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/bqlbs.png">
                  </span>
                  <span class="link">
                    <a href="http://www.urselect.com/" target="_blank" rel="noopener">优社电商</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8s88c.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yuanrenxue.com/" target="_blank" rel="noopener">猿人学</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2wgg5.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.yunlifang.cn/" target="_blank" rel="noopener">云立方</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/shwr6.png">
                  </span>
                  <span class="link">
                    <a href="http://lanbing510.info/" target="_blank" rel="noopener">冰蓝</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/blvoh.jpg">
                  </span>
                  <span class="link">
                    <a href="https://lengyue.me/" target="_blank" rel="noopener">冷月</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="http://qianxunclub.com/favicon.png">
                  </span>
                  <span class="link">
                    <a href="http://qianxunclub.com/" target="_blank" rel="noopener">千寻啊千寻</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/0044u.jpg">
                  </span>
                  <span class="link">
                    <a href="http://kodcloud.com/" target="_blank" rel="noopener">可道云</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ygnpn.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.kunkundashen.cn/" target="_blank" rel="noopener">坤坤大神</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/22uv1.png">
                  </span>
                  <span class="link">
                    <a href="http://www.cenchong.com/" target="_blank" rel="noopener">岑冲博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ev9kl.png">
                  </span>
                  <span class="link">
                    <a href="http://www.zxiaoji.com/" target="_blank" rel="noopener">张小鸡</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.503error.com/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.503error.com/" target="_blank" rel="noopener">张志明个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/x714o.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.hubwiz.com/" target="_blank" rel="noopener">汇智网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/129d8.png">
                  </span>
                  <span class="link">
                    <a href="https://www.bysocket.com/" target="_blank" rel="noopener">泥瓦匠BYSocket</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://www.xiongge.club/favicon.ico">
                  </span>
                  <span class="link">
                    <a href="https://www.xiongge.club/" target="_blank" rel="noopener">熊哥club</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/3w4fe.png">
                  </span>
                  <span class="link">
                    <a href="https://zerlong.com/" target="_blank" rel="noopener">知语</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/44hxf.png">
                  </span>
                  <span class="link">
                    <a href="http://redstonewill.com/" target="_blank" rel="noopener">红色石头</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/8g1fk.jpg">
                  </span>
                  <span class="link">
                    <a href="http://www.laodong.me/" target="_blank" rel="noopener">老董博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/wkaus.jpg">
                  </span>
                  <span class="link">
                    <a href="https://zhaoshuai.me/" target="_blank" rel="noopener">碎念</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/pgo0r.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.chenwenguan.com/" target="_blank" rel="noopener">陈文管的博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/kk82a.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.lxlinux.net/" target="_blank" rel="noopener">良许Linux教程网</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/lj0t2.jpg">
                  </span>
                  <span class="link">
                    <a href="https://tanqingbo.cn/" target="_blank" rel="noopener">IT码农</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/i8cdr.png">
                  </span>
                  <span class="link">
                    <a href="https://junyiseo.com/" target="_blank" rel="noopener">均益个人博客</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/chwv2.png">
                  </span>
                  <span class="link">
                    <a href="https://brucedone.com/" target="_blank" rel="noopener">大鱼的鱼塘</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/2y43o.png">
                  </span>
                  <span class="link">
                    <a href="http://bbs.nightteam.cn/" target="_blank" rel="noopener">夜幕爬虫安全论坛</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/zvc3w.jpg">
                  </span>
                  <span class="link">
                    <a href="https://www.weishidong.com/" target="_blank" rel="noopener">韦世东的技术专栏</a>
                  </span>
                </li>
                <li class="friend">
                  <span class="logo">
                    <img src="https://qiniu.cuiqingcai.com/ebudy.jpg">
                  </span>
                  <span class="link">
                    <a href="https://chuanjiabing.com/" target="_blank" rel="noopener">穿甲兵技术社区</a>
                  </span>
                </li>
              </ul>
            </div>
          </div>
        </aside>
        <div id="sidebar-dimmer"></div>
      </div>
    </main>
    <footer class="footer">
      <div class="footer-inner">
        <div class="copyright"> &copy; <span itemprop="copyrightYear">2021</span>
          <span class="with-love">
            <i class="fa fa-heart"></i>
          </span>
          <span class="author" itemprop="copyrightHolder">崔庆才丨静觅</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-chart-area"></i>
          </span>
          <span title="站点总字数">2.6m</span>
          <span class="post-meta-divider">|</span>
          <span class="post-meta-item-icon">
            <i class="fa fa-coffee"></i>
          </span>
          <span title="站点阅读时长">39:54</span>
        </div>
        <div class="powered-by">由 <a href="https://hexo.io/" class="theme-link" rel="noopener" target="_blank">Hexo</a> & <a href="https://pisces.theme-next.org/" class="theme-link" rel="noopener" target="_blank">NexT.Pisces</a> 强力驱动 </div>
        <div class="beian"><a href="https://beian.miit.gov.cn/" rel="noopener" target="_blank">京ICP备18015597号-1 </a>
        </div>
        <script>
          (function ()
          {
            function leancloudSelector(url)
            {
              url = encodeURI(url);
              return document.getElementById(url).querySelector('.leancloud-visitors-count');
            }

            function addCount(Counter)
            {
              var visitors = document.querySelector('.leancloud_visitors');
              var url = decodeURI(visitors.id);
              var title = visitors.dataset.flagTitle;
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                if (results.length > 0)
                {
                  var counter = results[0];
                  leancloudSelector(url).innerText = counter.time + 1;
                  Counter('put', '/classes/Counter/' + counter.objectId,
                  {
                    time:
                    {
                      '__op': 'Increment',
                      'amount': 1
                    }
                  }).catch(error =>
                  {
                    console.error('Failed to save visitor count', error);
                  });
                }
                else
                {
                  Counter('post', '/classes/Counter',
                  {
                    title,
                    url,
                    time: 1
                  }).then(response => response.json()).then(() =>
                  {
                    leancloudSelector(url).innerText = 1;
                  }).catch(error =>
                  {
                    console.error('Failed to create', error);
                  });
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }

            function showTime(Counter)
            {
              var visitors = document.querySelectorAll('.leancloud_visitors');
              var entries = [...visitors].map(element =>
              {
                return decodeURI(element.id);
              });
              Counter('get', '/classes/Counter?where=' + encodeURIComponent(JSON.stringify(
              {
                url:
                {
                  '$in': entries
                }
              }))).then(response => response.json()).then((
              {
                results
              }) =>
              {
                for (let url of entries)
                {
                  let target = results.find(item => item.url === url);
                  leancloudSelector(url).innerText = target ? target.time : 0;
                }
              }).catch(error =>
              {
                console.error('LeanCloud Counter Error', error);
              });
            }
            let
            {
              app_id,
              app_key,
              server_url
            } = {
              "enable": true,
              "app_id": "6X5dRQ0pnPWJgYy8SXOg0uID-gzGzoHsz",
              "app_key": "ziLDVEy73ne5HtFTiGstzHMS",
              "server_url": "https://6x5drq0p.lc-cn-n1-shared.com",
              "security": false
            };

            function fetchData(api_server)
            {
              var Counter = (method, url, data) =>
              {
                return fetch(`${api_server}/1.1${url}`,
                {
                  method,
                  headers:
                  {
                    'X-LC-Id': app_id,
                    'X-LC-Key': app_key,
                    'Content-Type': 'application/json',
                  },
                  body: JSON.stringify(data)
                });
              };
              if (CONFIG.page.isPost)
              {
                if (CONFIG.hostname !== location.hostname) return;
                addCount(Counter);
              }
              else if (document.querySelectorAll('.post-title-link').length >= 1)
              {
                showTime(Counter);
              }
            }
            let api_server = app_id.slice(-9) !== '-MdYXbMMI' ? server_url : `https://${app_id.slice(0, 8).toLowerCase()}.api.lncldglobal.com`;
            if (api_server)
            {
              fetchData(api_server);
            }
            else
            {
              fetch('https://app-router.leancloud.cn/2/route?appId=' + app_id).then(response => response.json()).then((
              {
                api_server
              }) =>
              {
                fetchData('https://' + api_server);
              });
            }
          })();

        </script>
      </div>
      <div class="footer-stat">
        <span id="cnzz_stat_icon_1279355174"></span>
        <script type="text/javascript">
          document.write(unescape("%3Cspan id='cnzz_stat_icon_1279355174'%3E%3C/span%3E%3Cscript src='https://v1.cnzz.com/z_stat.php%3Fid%3D1279355174%26online%3D1%26show%3Dline' type='text/javascript'%3E%3C/script%3E"));

        </script>
      </div>
    </footer>
  </div>
  <script src="//cdn.jsdelivr.net/npm/animejs@3.2.1/lib/anime.min.js"></script>
  <script src="//cdn.jsdelivr.net/npm/pangu@4/dist/browser/pangu.min.js"></script>
  <script src="/js/utils.js"></script>
  <script src="/.js"></script>
  <script src="/js/schemes/pisces.js"></script>
  <script src="/.js"></script>
  <script src="/js/next-boot.js"></script>
  <script src="/.js"></script>
  <script>
    (function ()
    {
      var canonicalURL, curProtocol;
      //Get the <link> tag
      var x = document.getElementsByTagName("link");
      //Find the last canonical URL
      if (x.length > 0)
      {
        for (i = 0; i < x.length; i++)
        {
          if (x[i].rel.toLowerCase() == 'canonical' && x[i].href)
          {
            canonicalURL = x[i].href;
          }
        }
      }
      //Get protocol
      if (!canonicalURL)
      {
        curProtocol = window.location.protocol.split(':')[0];
      }
      else
      {
        curProtocol = canonicalURL.split(':')[0];
      }
      //Get current URL if the canonical URL does not exist
      if (!canonicalURL) canonicalURL = window.location.href;
      //Assign script content. Replace current URL with the canonical URL
      ! function ()
      {
        var e = /([http|https]:\/\/[a-zA-Z0-9\_\.]+\.baidu\.com)/gi,
          r = canonicalURL,
          t = document.referrer;
        if (!e.test(r))
        {
          var n = (String(curProtocol).toLowerCase() === 'https') ? "https://sp0.baidu.com/9_Q4simg2RQJ8t7jm9iCKT-xh_/s.gif" : "//api.share.baidu.com/s.gif";
          t ? (n += "?r=" + encodeURIComponent(document.referrer), r && (n += "&l=" + r)) : r && (n += "?l=" + r);
          var i = new Image;
          i.src = n
        }
      }(window);
    })();

  </script>
  <script src="/js/local-search.js"></script>
  <script src="/.js"></script>
</body>

</html>
